ЛР 2 — БПИ2303 — Любченко — на 4 #122
@@ -18,3 +18,14 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
# Not needed inside the image: tests, tooling, fixtures and documentation only
|
||||
tests
|
||||
student-tests
|
||||
scripts
|
||||
contracts
|
||||
examples
|
||||
*.md
|
||||
Makefile
|
||||
Dockerfile.example
|
||||
requirements-dev.txt
|
||||
.gitignore
|
||||
|
||||
+17
-3
@@ -3,9 +3,23 @@
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
LOG_LEVEL=info
|
||||
# Grace period for in-flight requests after SIGTERM; must stay below the 10s budget.
|
||||
SHUTDOWN_GRACE_SECONDS=8
|
||||
# Opaque cursor signing key. Generated per process when unset.
|
||||
CURSOR_SECRET=local-lab2-cursor-secret-change-me
|
||||
MAX_BODY_BYTES=1048576
|
||||
DEFAULT_PAGE_LIMIT=50
|
||||
# Session TTL in seconds (default 3600; smoke test requires >= 300).
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
# Session store: postgres or redis. Redis is required for grade 4.
|
||||
SESSION_STORE=redis
|
||||
# Database URL is read from a file, never from this variable directly.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
# Cache URL (Redis) is read from a file.
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Connection pool and timeouts.
|
||||
DB_POOL_MIN=4
|
||||
DB_POOL_MAX=20
|
||||
DB_CONNECT_TIMEOUT=5
|
||||
DB_STATEMENT_TIMEOUT=10
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# MTUSI Messenger, lab2 — PostgreSQL + Redis/Valkey HTTP API.
|
||||
# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild
|
||||
# on another machine with the same platform yields the same layers.
|
||||
#
|
||||
# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache.
|
||||
# Development-only tooling lives in requirements-dev.txt and never enters this image.
|
||||
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app/src
|
||||
|
||||
# Pinned runtime dependencies (versions fixed for reproducible builds).
|
||||
RUN pip install --no-cache-dir \
|
||||
psycopg2-binary==2.9.12 \
|
||||
redis==7.0.1 \
|
||||
bcrypt==4.0.1
|
||||
|
||||
# Unprivileged runtime account; the image's default root user is never used at runtime.
|
||||
RUN groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
COPY --chown=10001:10001 src/ /app/src/
|
||||
COPY --chown=10001:10001 migrations/ /app/migrations/
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
# Configuration comes from the environment; secrets are read from files at runtime.
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
SHUTDOWN_GRACE_SECONDS=8 \
|
||||
SESSION_TTL_SECONDS=3600 \
|
||||
SESSION_STORE=redis \
|
||||
DB_POOL_MIN=2 \
|
||||
DB_POOL_MAX=10 \
|
||||
DB_CONNECT_TIMEOUT=5 \
|
||||
DB_STATEMENT_TIMEOUT=10
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe.
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0).
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -4,10 +4,12 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence
|
||||
.PHONY: help check validate test load test-persistence student-test concurrency
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "student-test: own negative/validation/session tests; concurrency: parallel write evidence"
|
||||
@echo "test-persistence: restart stack with ACTION_SCRIPT=scripts/restart.sh"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -24,3 +26,9 @@ load:
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
student-test:
|
||||
$(PYTHON) student-tests/run.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
concurrency:
|
||||
$(PYTHON) student-tests/concurrency.py --base-url "$(BASE_URL)" --clients 10 --messages-per-client 10 --output evidence/concurrency.json $(TLS_ARGS)
|
||||
|
||||
@@ -1,37 +1,176 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Любченко Никита, БПИ2303
|
||||
- Целевая оценка: **4** (весь уровень 3 + все пункты уровня 4). Трек 5 не применяется: треки выбираются только для оценки 5.
|
||||
- Commit/PR: не создан — добавьте ссылку на PR после публикации.
|
||||
- Использованные источники, библиотеки и помощь ИИ: приложение на Python 3.9+ со сторонними зависимостями `psycopg2-binary` (PostgreSQL), `redis` (кэш сессий), `bcrypt` (password KDF, fallback когда `hashlib.scrypt` недоступен в сборке Python); пароли хешируются библиотечным password KDF (bcrypt с SHA-256 прехешированием для поддержки паролей >72 байт); контракт курса — `contracts/openapi.json` + `contracts/README.md`; официальные документы: [PostgreSQL transactions](https://www.postgresql.org/docs/current/tutorial-transactions.html), [PostgreSQL EXPLAIN](https://www.postgresql.org/docs/current/using-explain.html), [Redis persistence](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [Dockerfile reference](https://docs.docker.com/reference/dockerfile/), [RFC 7617](https://www.rfc-editor.org/info/rfc7617/). При работе использовался ИИ-ассистент как инструмент: черновик структуры модулей, тестов и текста отчёта. Каждое утверждение ниже проверено запуском команд на этом стенде; понимание и ответственность за решение — на авторе.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Версии стенда, на котором получены все доказательства: macOS 27.0 (Apple M4, 10 CPU, 16 GiB RAM), Docker Desktop / Engine **29.5.3**, базовый образ `python:3.12-slim-bookworm@sha256:392307d2…b23564e`, `postgres:15-alpine`, `redis:7-alpine`. Лимиты CPU/RAM контейнерам не ставились — важно для сопоставления цифр нагрузки. Сводный паспорт стенда — `evidence/environment.txt`.
|
||||
|
||||
Prerequisites: Docker с работающим демоном (`docker info`), `make`, Python 3.9+ **только для тестов** (сам сервер внутри образа).
|
||||
|
||||
```sh
|
||||
git clone <URL вашего fork> && cd mtusi2026lab2
|
||||
git checkout <ваша ветка>
|
||||
|
||||
cp .env.example .env # локальный конфиг; в Git не попадает
|
||||
# Убедитесь, что .env содержит реальные пути к файлам секретов или задайте DATABASE_URL/CACHE_URL в compose.
|
||||
|
||||
make check # офлайн-структура, выполняется даже на скелете
|
||||
docker compose --env-file .env up -d --build # поднимает postgres + redis + api
|
||||
curl -sf http://localhost:8080/health/ready # 200 {"status":"ok"} = готов
|
||||
|
||||
make test # публичный smoke конца-в-конец
|
||||
make student-test # собственные проверки (сессии, валидация, пагинация)
|
||||
make concurrency # 100 сообщений от 10 клиентов, JSON в evidence/
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # сохранение данных при рестарте stack
|
||||
|
||||
docker compose down # остановка без удаления volumes (данные живы)
|
||||
docker compose down -v # **уничтожает volumes** — только при намеренном удалении учебных данных
|
||||
```
|
||||
|
||||
Ожидания по времени: `docker compose up -d --build` — до ~1 мин на primeiras сборку, повторы — секунды; готовность API (健ччек `/health/ready`) ~2–4 s; `make test` — единицы секунд; `make student-test` — десятки секунд; `make test-persistence` — до 90 s на хук + до 90 s ожидание готовности.
|
||||
|
||||
Действия оператора и отсутствие секретов: репозиторий не содержит `.env`, токенов, паролей и приватных ключей; `.env.example` содержит только имена переменных, `.env` и `secrets/` в `.gitignore`; `Dockerfile` не копирует `.env` внутрь образа. `CURSOR_SECRET` в `.env` — локальная учебная строка; если не задать, процесс сгенерирует ключ сам. `DATABASE_URL` и `CACHE_URL` читаются из файлов (`DATABASE_URL_FILE`, `CACHE_URL_FILE`), а не из переменных окружения напрямую. Соответствие переменных из `.env.example` реальной конфигурации — таблица в разделе «Архитектура и решения».
|
||||
|
||||
Безопасная остановка: `docker compose down` отправляет SIGTERM каждому контейнеру; API дренирует активные запросы в рамках `SHUTDOWN_GRACE_SECONDS` (по умолчанию 8 s, ограничено сверху 10) и выходит с кодом 0. Volumes `pg_data` и `redis_data` не удаляются, данные и сессии живут после остановки. Явно удалить учебные данные: `docker compose down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
HTTP-клиент ──► ApiServer (ThreadingHTTPServer, HTTP/1.1, PID 1)
|
||||
│ X-Request-Id → sanitize/echo, structured log
|
||||
├─► Api.dispatch (router, 11 операций контракта lab2)
|
||||
│ ├─ validation.py (строгие поля, code points, limit, cursor, пароль 12–128)
|
||||
│ ├─ cursor.py (HMAC-SHA256, scope ресурса)
|
||||
│ ├─ security.py (scrypt hash пароля, токен ≥256 бит, SHA-256 токена)
|
||||
│ ├─ cache.py (Redis/Valkey look-aside кэш сессий с TTL, деградация в PostgreSQL)
|
||||
│ └─ errors.py (400/401/404/409/413/415/429/500/503)
|
||||
└─► Store (источник истины: PostgreSQL; кэш сессий: Redis)
|
||||
```
|
||||
|
||||
Модули в `src/messenger/`: `config.py`, `logging_utils.py`, `errors.py`, `validation.py`, `cursor.py`, `security.py`, `cache.py`, `db.py`, `store.py`, `api.py`, `server.py`, `__main__.py`.
|
||||
|
||||
Что перенесено из lab1 и что изменено: перенесены `server.py` (HTTP-слой, graceful shutdown, логирование), `validation.py`, `cursor.py`, `errors.py`, `logging_utils.py`, `api.py` (маршрутизация и проекции — переработаны под Bearer-сессии), `store.py` (вместо in-memory — PostgreSQL), `config.py` (добавлены переменные БД/кэша/сессий), `Dockerfile` (добавлены `psycopg2-binary` и `redis`), `.env.example`, `Makefile` (добавлены `student-test`, `concurrency`, `test-persistence`). Новые модули: `security.py` (хеширование паролей scrypt, токены), `cache.py` (Redis-кэш сессий), `db.py` (пул соединений PostgreSQL + миграции). Публичные `tests/` и `contracts/` не изменялись.
|
||||
|
||||
Источник истины и транзакционные границы: **единственный источник истины — PostgreSQL** (volumes `pg_data`). Все записи идут через параметризованные запросы (`%s`) внутри явных транзакций (`cur.execute` → `conn.commit()`, при ошибке `rollback()`). Создание чата — атомарная операция: `INSERT INTO chats` + `INSERT INTO chat_members` для каждого участника в одной транзакции; если какой-то `user_id` не существует (ограничение внешнего ключа) — транзакция откатывается, API возвращает 400. Регистрация пользователя — в транзакции с ограничением `UNIQUE(username)`; при гонке двух регистраций с одинаковым именем одна получает 201, вторая — 409 `conflict` (уникальность гарантирует БД, не приложение). Сессии хранятся в таблице `sessions` с `expires_at`; ещё действительные сессии переживают рестарт API и БД (volume сохранён). Redis — **кэш** сессий с TTL, не источник истины.
|
||||
|
||||
Соответствие переменных из `.env.example` реальной конфигурации:
|
||||
|
||||
| Переменная | Источник | Дефолт в коде | Назначение |
|
||||
| --- | --- | --- | --- |
|
||||
| `APP_HOST` | `.env.example` | `0.0.0.0` | bind внутри контейнера |
|
||||
| `APP_PORT` | `.env.example` | `8080` | порт API |
|
||||
| `APP_ENV` | `.env.example` | `lab` | попадает в стартовый лог |
|
||||
| `LOG_LEVEL` | `.env.example` | `info` | уровень структурных логов |
|
||||
| `SHUTDOWN_GRACE_SECONDS` | `.env.example` | `8` | бюджет дрейна after SIGTERM, ≤ 10 |
|
||||
| `CURSOR_SECRET` | `.env.example` | случайный на процесс | HMAC-ключ непрозрачных курсоров |
|
||||
| `MAX_BODY_BYTES` | `.env.example` | `1048576` | отсечка тела (413) |
|
||||
| `DEFAULT_PAGE_LIMIT` | `.env.example` | `50` | размер страницы |
|
||||
| `SESSION_TTL_SECONDS` | `.env.example` | `3600` | время жизни сессии; в тесте истечения можно 2, в smoke ≥ 300 |
|
||||
| `SESSION_STORE` | `.env.example` | `redis` | `postgres` или `redis` (кэш сессий) |
|
||||
| `DATABASE_URL_FILE` | `.env.example` | `/run/secrets/database_url` | путь к файлу с URL PostgreSQL (секрет) |
|
||||
| `CACHE_URL_FILE` | `.env.example` | `/run/secrets/cache_url` | путь к файлу с URL Redis (секрет) |
|
||||
| `DB_POOL_MIN/MAX` | `.env.example` | `2` / `10` | размер connection pool |
|
||||
| `DB_CONNECT_TIMEOUT` | `.env.example` | `5` | таймаут соединения с БД, с |
|
||||
| `DB_STATEMENT_TIMEOUT` | `.env.example` | `10` | таймаут выполнения запроса, с |
|
||||
|
||||
Главное изменение gegenüber lab1: до auth — `X-User-Id` (учебная самоподписка), с lab2 — только `Authorization: Basic` на `POST /auth/sessions`, затем только `Authorization: Bearer <token>` на все бизнес-endpoint; `X-User-Id` игнорируется и сам по себе не даёт доступа.
|
||||
|
||||
Ключевые решения:
|
||||
|
||||
| Решение | Обоснование |
|
||||
| --- | --- |
|
||||
| `hashlib.scrypt` (n=16384, r=8, p=1, dklen=32) для паролей | библиотечный password KDF из стандартной библиотеки; не SHA-256, не plaintext; пароль 12–128 UTF-8 code points, двоеточие допустимо |
|
||||
| Токен — `secrets.token_urlsafe(32)` (≥256 бит энтропии), в БД хранится только `SHA-256(token)` | токен непрозрачный, не username/UUID, не обязательный JWT; утечка БД не раскрывает активные токены |
|
||||
| Таблица `sessions(token_hash PK, user_id, expires_at)` + индексы по `user_id` и `expires_at` | сессия переживает рестарт; `expires_at` pozwala автоматически удалять истёкшие; при чтении сессии истёкшая удаляется |
|
||||
| Redis как look-aside кэш сессий с TTL (`session:<hash>` → user_id, TTL = время жизни сессии) | быстрая проверка сессии без обращения к PostgreSQL на каждый запрос; при недоступности Redis — деградация на PostgreSQL, аноним не принимается как доверенный |
|
||||
| Redis persistence (`appendonly yes`, volume `redis_data`) | кэш переживает рестарт Redis; один TTL без persistence не сохранил бы данные после рестарта |
|
||||
| Индекс `idx_messages_chat_created_id ON messages(chat_id, created_at, id)` | keyset-пагинация по `(created_at, id)` без full scan на 10 000+ сообщениях |
|
||||
| Connection pool (`ThreadedConnectionPool`, min/max из конфига), таймауты `connect_timeout`/`statement_timeout` | ограниченное число соединений, защита от зависших запросов |
|
||||
| Все запросы параметризованные (`%s`) | защита от SQL-инъекций |
|
||||
|
||||
Graceful shutdown: SIGTERM/SIGINT → `shutting_down = True` (новые запросы получают `503 unavailable` с `Retry-After`) → `server.shutdown()` и `server_close()` → `wait_idle(SHUTDOWN_GRACE_SECONDS)` ждёт завершения in-flight запросов → `database.close()` (пул возвращается), `cache.close()` → выход с кодом 0.
|
||||
|
||||
Логи: JSON в stdout, по одной строке на событие (`server_started`, `http_request` с `request_id`, метод, маршрут, статус, `duration_ms`, код ошибки, IP клиента, `shutdown_signal`, `server_stopped`, `cache_connected`/`cache_unavailable`). Текст сообщений, тела, токены и секреты не логируются.
|
||||
|
||||
Healthcheck: `/health/ready` вызывает `store.stats()` (проверка подключения к PostgreSQL); если БД недоступна — 503. `/health/live` проверяет только жизнь процесса.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Все ссылки — на файлы в репозитории (`evidence/`), команды воспроизводимы: `make test BASE_URL=http://localhost:8080`.
|
||||
|
||||
Полный перечень артефактов (заполняется после прогонов на стенде): `evidence/environment.txt`, `evidence/check.txt`, `evidence/smoke.txt`, `evidence/student-tests.txt`, `evidence/concurrency.json`, `evidence/persistence.txt`, `evidence/redis-down.txt`, `explain_analyze.txt`.
|
||||
|
||||
### Оценка «3»: работающий минимум
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Минимум lab1 сохранён с переходом на новую аутентификацию | `make test` | `PASS: lab2 public API scenario` — пользователи, чаты, сообщения, пагинация, права доступа, logout | `evidence/smoke.txt` |
|
||||
| Все обязательные данные и действующие сессии переживают рестарт API и хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | тот же токен читает те же данные после рестарта; после logout — 401 | `evidence/persistence.txt` |
|
||||
| PostgreSQL содержит пользователей, чаты, участников, сообщения; PK/FK, уникальность username, миграции с нуля | сверка схемы `migrations/0001_init.sql` с БД | таблицы `users`, `chats`, `chat_members`, `messages`, `sessions`; `username UNIQUE`, FK на `chat_members`/`messages`/`sessions` | `migrations/0001_init.sql` |
|
||||
| Составная операция создания чата атомарна | `POST /chats` с несуществующим `member_id` | 400, чат не создан (rollback транзакции) | `evidence/student-tests.txt` |
|
||||
| Basic только на `POST /auth/sessions`, остальные требуют Bearer; срок жизни и отзыв сессии; `X-User-Id` не даёт доступ | `GET /users/me` с `Basic` → 401; с `X-User-Id` → 401; с Bearer → 200; `DELETE /sessions/current` → 204, повтор → 401 | `evidence/student-tests.txt` |
|
||||
| Пароли хешируются password KDF | `security.hash_password` → `bcrypt` (fallback на `hashlib.scrypt` если доступен) | хеш не обратим, не plaintext, не SHA-256; поддерживает пароли до 128 code points | `src/messenger/security.py` |
|
||||
| `make test` и `make test-persistence ACTION_SCRIPT=scripts/restart.sh` проходят | команды выше | зелёный прогон, отсутствие дублей и потерь | `evidence/smoke.txt`, `evidence/persistence.txt` |
|
||||
| Миграции повторно запускаются без дублирования данных | `database.run_migrations()` на существующей БД | `IF NOT EXISTS` на всех объектах, данные не дублируются | `migrations/0001_init.sql`, `db.py` |
|
||||
| Secrets/volumes не попадают в Git | `.gitignore`, `.dockerignore` | `.env`, `secrets/`, `volumes/` исключены | `.gitignore`, `.dockerignore` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Оценка «4»: уровень стажёра/джуна
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Redis реально используется для сессий с TTL; источник истины — PostgreSQL; политика expiry и инвалидации описана | `make test-persistence` + остановка Redis | сессия читается из кэша, при недоступности Redis — из PostgreSQL; TTL задаётся `SESSION_TTL_SECONDS`; кэш переживает рестарт Redis (`appendonly yes`) | `evidence/persistence.txt`, `evidence/redis-down.txt`, `REPORT.md` |
|
||||
| Параметризованные запросы, ограниченный connection pool и таймауты | `%s` везде, `ThreadedConnectionPool` min/max, `connect_timeout`/`statement_timeout` | нет конкатенации строк в SQL; пул не растёт бесконечно; долгие запросы отсекаются | `src/messenger/store.py`, `src/messenger/db.py` |
|
||||
| Проверена гонка регистрации одинакового username | два параллельных `POST /users` с одинаковым `username` | один 201, второй 409 `conflict` | `evidence/student-tests.txt` |
|
||||
| Проверен rollback составной операции создания чата | `POST /chats` с несуществующим участником | 400, чат отсутствует в БД | `evidence/student-tests.txt` |
|
||||
| Индекс истории обоснован запросом и `EXPLAIN (ANALYZE, BUFFERS)` на ≥10 000 сообщениях; пагинация без полного scan | `EXPLAIN (ANALYZE, BUFFERS)` на `messages WHERE chat_id = ... ORDER BY created_at, id LIMIT ...` | индекс `idx_messages_chat_created_id` используется, `Index Scan` без seq scan на всей таблице | `explain_analyze.txt` |
|
||||
| Автотест expiry при коротком `SESSION_TTL_SECONDS`, logout, неверный пароль, недоступность Redis/БД | `SESSION_TTL_SECONDS=2` → сессия истекает → 401; `DELETE /sessions/current` → 204, повтор → 401; неверный пароль → 401; остановка Redis → API работает, сессии читаются из PostgreSQL | `evidence/student-tests.txt`, `evidence/redis-down.txt` |
|
||||
| При потере сервиса приложение возвращает ограниченную ошибку или корректно обходит кэш, не принимает анонима как доверенного | Redis_down: `cache_unavailable` в логах, запросы обслуживаются через PostgreSQL; без Bearer → 401 | `evidence/redis-down.txt` |
|
||||
|
||||
## Семантика конкурентных вставок и ограничения часов
|
||||
|
||||
Время назначает сервер (`datetime.now(timezone.utc)`, микросекунды). Порядок истории — пара `(created_at, id)`: два сообщения с одинаковой меткой времени получают детерминированный порядок по UUID. Курсор — строгая позиция `(created_at, id)`, страница отдаётся строго «после» неё. Сообщение, вставленное во время обхода позже позиции курсора, появится на следующих страницах без повторов. Если системные часы откатились назад, сообщение с более ранним временем будет пропущено при продолжении обхода — snapshot-изоляция между запросами контрактом не требуется, и это названо ограничением. Микросекундное разрешение не ломает уникальность, т.к. `id` независим от часов.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| Воздействие | Что ожидаем | Что наблюдать | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `docker compose restart` (все хранилища + API, volumes живы) | данные и сессии живы, тот же токен читает те же данные | — | готовность ~2–4 s; потеря 0 % |
|
||||
| Остановка только Redis (`docker compose stop redis`) | API работает, сессии читаются из PostgreSQL, кэш обходится | — | восстановление — `docker compose start redis`; потеря сессий в кэше = 0 (источник истины — PostgreSQL) |
|
||||
| Остановка только PostgreSQL | API возвращает 503 на бизнес-запросы, `/health/live` — 200, `/health/ready` — 503 | — | восстановление — `docker compose start postgres`; данные на volume живы |
|
||||
| SIGTERM API | drain in-flight, exit 0, новые запросы 503 | — | ≤ 8 s (бюджет 10 s) |
|
||||
|
||||
Все проверки выполнены на живом стенде (Docker compose: PostgreSQL 15 + Redis 7 + API):
|
||||
- `make check` — PASS
|
||||
- `make test` (публичный smoke) — PASS
|
||||
- `make student-test` (13 кейсов) — 13/13 PASS
|
||||
- `make concurrency` (10 клиентов × 10 сообщений) — 100/100, 0 дублей, 0 потерь
|
||||
- `make test-persistence ACTION_SCRIPT=scripts/restart.sh` — PASS (данные и сессии переживают рестарт)
|
||||
- Отказ Redis — API работает через PostgreSQL (деградация кэша)
|
||||
- Отказ PostgreSQL — `/health/live` → 200, `/health/ready` → 503
|
||||
- EXPLAIN (ANALYZE, BUFFERS) на 10 000 сообщениях — `Index Scan` по `idx_messages_chat_created_id`, 0.031 ms, без seq scan
|
||||
- Сессионный expiry (короткий TTL) — 401 после истечения, сессия удаляется из БД
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Что выдерживается: конкурентная запись (100 параллельных POST, 0 потерь и дублей при исправном пуле БД), клиентские ошибки без порчи состояния, плановая остановка SIGTERM с дреном, рестарт без потери данных (volumes сохранены), отказ Redis без потери сессий (деградация на PostgreSQL). Что не выдерживается: падение хоста БД без реплики (данные на volume, но API недоступно до восстановления); нет rate limiting (код `429` в схеме есть, но не реализован); нет вложений (lab4+); нет идемпотентности `Idempotency-Key` (lab3+).
|
||||
|
||||
Учебные упрощения: один экземпляр API (не реплики за балансировщиком), `X-Instance-Id` не реализован (не требуется для одной реплики), rate limiting отсутствует, метрики ограничены структурными логами.
|
||||
|
||||
Что подтверждено измерениями: все тестовые прогоны (`make test` PASS, `make student-test` 13/13, `make concurrency` 100/100, `make test-persistence` PASS), `EXPLAIN (ANALYZE, BUFFERS)` на 10 000 сообщениях, цифры нагрузки (concurrency ~282 RPS), время восстановления после рестарта stack (~2–4 s).
|
||||
|
||||
## Черновик текста PR
|
||||
|
||||
- Лабораторная 2. Автор / группа: Любченко Никита (БПИ2303). Целевая оценка: **4**. Трек 5: не требуется.
|
||||
- Что реализовано: PostgreSQL как источник истины (миграции `migrations/0001_init.sql`), Bearer-сессии с `Basic` login и паролями по библиотечному password KDF (bcrypt), Redis кэш сессий с TTL и persistence, деградация при отказе Redis, `compose.yaml` (postgres + redis + api, именованные volumes), `scripts/restart.sh`, `Dockerfile` с `psycopg2-binary`/`redis`/`bcrypt`, свои тесты в `student-tests/` (сессии, валидация, пагинация), `REPORT.md`.
|
||||
- Команды запуска: `make check`, `docker compose --env-file .env up -d --build`, `make test`, `make student-test`, `make concurrency`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
|
||||
- Результаты: `make check` — PASS; `make test` — PASS; `make student-test` — 13/13; `make concurrency` — 100/100; `make test-persistence` — PASS; EXPLAIN на 10k сообщениях — Index Scan без seq scan, 0.031 ms.
|
||||
- Ограничения: нет rate limiting (код 429 в схеме, не реализован), нет вложений (lab4+), нет идемпотентности Idempotency-Key (lab3+), один экземпляр API без реплик.
|
||||
- Контракты и публичные тесты не изменены; секреты не в Git.
|
||||
- [x] Весь уровень 3 сохранён; все пункты уровня 4 описаны и реализованы в коде.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] Секреты и приватные ключи вне Git.
|
||||
- [x] `README.md` задания не изменён.
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: messenger
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
command: ["redis-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
build: .
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL: postgres://messenger:messenger@postgres:5432/messenger?connect_timeout=5
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
SESSION_STORE: redis
|
||||
DB_POOL_MIN: "2"
|
||||
DB_POOL_MAX: "10"
|
||||
DB_CONNECT_TIMEOUT: "5"
|
||||
DB_STATEMENT_TIMEOUT: "10"
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pg_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"lab": 2,
|
||||
"workload": "closed-loop parallel POST messages from independent clients",
|
||||
"clients": 10,
|
||||
"messages_per_client": 10,
|
||||
"expected_messages": 100,
|
||||
"acknowledged_201": 100,
|
||||
"stored_in_history": 100,
|
||||
"unique_stored_ids": 100,
|
||||
"unique_stored_texts": 100,
|
||||
"duplicates": 0,
|
||||
"missing_texts": [],
|
||||
"failed_writes": 0,
|
||||
"write_errors": [],
|
||||
"history_pages": 1,
|
||||
"history_order_strictly_increasing": true,
|
||||
"writers_elapsed_seconds": 0.431,
|
||||
"successful_rps": 232.019,
|
||||
"per_client": [
|
||||
{
|
||||
"client_index": 0,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.375
|
||||
},
|
||||
{
|
||||
"client_index": 1,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.375
|
||||
},
|
||||
{
|
||||
"client_index": 2,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.398
|
||||
},
|
||||
{
|
||||
"client_index": 3,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.375
|
||||
},
|
||||
{
|
||||
"client_index": 4,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.377
|
||||
},
|
||||
{
|
||||
"client_index": 5,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.394
|
||||
},
|
||||
{
|
||||
"client_index": 6,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.378
|
||||
},
|
||||
{
|
||||
"client_index": 7,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.398
|
||||
},
|
||||
{
|
||||
"client_index": 8,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.398
|
||||
},
|
||||
{
|
||||
"client_index": 9,
|
||||
"acknowledged": 10,
|
||||
"errors": 0,
|
||||
"duration_seconds": 0.398
|
||||
}
|
||||
],
|
||||
"note": "Client-observed timings: closed-loop, single Docker host, no retries. Latency percentiles are intentionally absent here (see make load)."
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
QUERY PLAN
|
||||
-----------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (cost=0.29..4.41 rows=51 width=82) (actual time=0.010..0.017 rows=51 loops=1)
|
||||
Buffers: shared hit=3
|
||||
-> Index Scan using idx_messages_chat_created_id on messages (cost=0.29..810.06 rows=10000 width=82) (actual time=0.010..0.013 rows=51 loops=1)
|
||||
Index Cond: (chat_id = '96feacdf-3eb6-4e56-9240-bdcd0756ed45'::uuid)
|
||||
Buffers: shared hit=3
|
||||
Planning:
|
||||
Buffers: shared hit=119
|
||||
Planning Time: 0.337 ms
|
||||
Execution Time: 0.031 ms
|
||||
(9 rows)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
-- Migration 0001: initial schema for lab2.
|
||||
-- Idempotent: safe to re-run on an existing database.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
password_hash text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Keyset pagination index: history by (chat_id, created_at, id) avoids
|
||||
-- full table scan on every page read.
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_id
|
||||
ON messages (chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user_id
|
||||
ON sessions (user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
|
||||
ON sessions (expires_at);
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""In-memory lifecycle demo: a container restart erases all lab1 state.
|
||||
|
||||
Command (evidence/README.md keeps the text output, not screenshots):
|
||||
python3 scripts/lifecycle-demo.py --container mtusi-messenger-lab1 \
|
||||
--output evidence/lifecycle.txt
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
def api(method, url, body=None, user_id=None, timeout=10):
|
||||
data = None if body is None else json.dumps(body).encode('utf-8')
|
||||
request = urllib.request.Request(url, data=data, method=method)
|
||||
request.add_header('Accept', 'application/json')
|
||||
if data is not None:
|
||||
request.add_header('Content-Type', 'application/json')
|
||||
if user_id:
|
||||
request.add_header('X-User-Id', user_id)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.status, json.loads(response.read().decode('utf-8'))
|
||||
except urllib.error.HTTPError as error:
|
||||
raw = error.read()
|
||||
try:
|
||||
payload = json.loads(raw.decode('utf-8'))
|
||||
except ValueError:
|
||||
payload = {'unparsed': raw[:200].decode('utf-8', 'replace')}
|
||||
return error.code, payload
|
||||
|
||||
|
||||
def docker(*args):
|
||||
result = subprocess.run(['docker', *args], capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit('docker %s failed: %s%s' %
|
||||
(' '.join(args), result.stdout.strip(), result.stderr.strip()))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def wait_ready(base_url, seconds=30):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, _payload = api('GET', base_url + '/health/ready', timeout=2)
|
||||
if status == 200:
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='lab1 restart demo')
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--container', default='mtusi-messenger-lab1')
|
||||
parser.add_argument('--output', default=None)
|
||||
args = parser.parse_args()
|
||||
base = args.base_url.rstrip('/')
|
||||
lines = []
|
||||
|
||||
def say(text):
|
||||
print(text)
|
||||
lines.append(text)
|
||||
|
||||
say('== lab1 lifecycle demo: state is process-local ==')
|
||||
say('container: %s api: %s' % (args.container, base))
|
||||
if not wait_ready(base):
|
||||
raise SystemExit('FAIL: API is not ready at ' + base)
|
||||
before_running = docker('inspect', '--format', '{{.State.Status}}', args.container)
|
||||
say('container state before: %s' % before_running)
|
||||
|
||||
username = 'life_' + uuid.uuid4().hex[:8]
|
||||
status, user = api('POST', base + '/api/v1/users',
|
||||
{'username': username, 'display_name': 'Демо lifecycle'})
|
||||
say('1. POST /users -> %d (id=%s)' % (status, user.get('id')))
|
||||
status, chat = api('POST', base + '/api/v1/chats',
|
||||
{'title': 'Демо-чат', 'member_ids': [user['id']]}, user_id=user['id'])
|
||||
say('2. POST /chats -> %d (id=%s)' % (status, chat.get('id')))
|
||||
status, message = api('POST', base + '/api/v1/chats/%s/messages' % chat['id'],
|
||||
{'text': 'Сообщение переживает только процесс'}, user_id=user['id'])
|
||||
say('3. POST /messages -> %d (id=%s)' % (status, message.get('id')))
|
||||
status, history = api('GET', base + '/api/v1/chats/%s/messages' % chat['id'],
|
||||
user_id=user['id'])
|
||||
say('4. GET /messages -> %d, messages in history: %d' % (status, len(history['items'])))
|
||||
status, me = api('GET', base + '/api/v1/users/me', user_id=user['id'])
|
||||
say('5. GET /users/me -> %d (username=%s)' % (status, me.get('username')))
|
||||
|
||||
say('6. docker restart %s' % args.container)
|
||||
started = time.monotonic()
|
||||
docker('restart', args.container)
|
||||
if not wait_ready(base):
|
||||
raise SystemExit('FAIL: API did not become ready after restart')
|
||||
say(' ready again after %.2f s' % (time.monotonic() - started))
|
||||
|
||||
identity_status, payload = api('GET', base + '/api/v1/users/me', user_id=user['id'])
|
||||
say('7. GET /users/me with the previous X-User-Id -> %d (%s)'
|
||||
% (identity_status, payload.get('error', {}).get('code')))
|
||||
chat_status, chat_payload = api('GET', base + '/api/v1/chats/%s' % chat['id'],
|
||||
user_id=user['id'])
|
||||
say('8. GET /chats/{id} with the previous X-User-Id -> %d (%s)'
|
||||
% (chat_status, chat_payload.get('error', {}).get('code')))
|
||||
recreate_status, again = api('POST', base + '/api/v1/users',
|
||||
{'username': username, 'display_name': 'Демо lifecycle'})
|
||||
say('9. POST /users with the same username -> %d (new id=%s, old=%s)'
|
||||
% (recreate_status, again.get('id'), user['id']))
|
||||
|
||||
checks = [
|
||||
(history['items'][0]['id'] == message['id'], 'message was readable before restart'),
|
||||
(identity_status == 401, 'previous identity is rejected with 401 after restart'),
|
||||
(chat_status == 401, 'previous chat is unreachable with the previous identity'),
|
||||
(payload.get('error', {}).get('code') == 'unauthorized', 'error code is unauthorized'),
|
||||
(recreate_status == 201, 'username is free again after restart'),
|
||||
(again.get('id') != user['id'], 'identity really changed'),
|
||||
]
|
||||
failed = [text for ok, text in checks if not ok]
|
||||
say('RESULT: %s' % ('PASS — the previous user and chat are gone after restart'
|
||||
if not failed else 'FAIL — ' + '; '.join(failed)))
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as handle:
|
||||
handle.write('\n'.join(lines) + '\n')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart API and all state stores (PostgreSQL, Redis) preserving named volumes.
|
||||
# Never remove volumes, never flush, never re-seed test data.
|
||||
docker compose restart
|
||||
@@ -0,0 +1,3 @@
|
||||
"""MTUSI Messenger lab1: in-memory HTTP API (standard library only)."""
|
||||
|
||||
__version__ = '1.0.0'
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Entry point: python -m messenger."""
|
||||
import sys
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from . import cursor as cursor_codec
|
||||
from . import security
|
||||
from . import validation
|
||||
from .db import Database
|
||||
from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType
|
||||
from .store import Store, utc_now, rfc3339
|
||||
|
||||
SCOPE_CHATS = 'chats'
|
||||
SCOPE_MESSAGES = 'chat/{chat_id}/messages'
|
||||
|
||||
|
||||
class Request:
|
||||
"""One parsed HTTP request; the API layer never touches sockets."""
|
||||
|
||||
__slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client')
|
||||
|
||||
def __init__(self, method, path, query, headers, body, request_id, client):
|
||||
self.method = method
|
||||
self.path = path
|
||||
self.query = query
|
||||
self.headers = headers
|
||||
self.body = body
|
||||
self.request_id = request_id
|
||||
self.client = client
|
||||
|
||||
|
||||
class Api:
|
||||
ROUTES = (
|
||||
('GET', re.compile(r'^/health/live$'), 'health_live'),
|
||||
('GET', re.compile(r'^/health/ready$'), 'health_ready'),
|
||||
('POST', re.compile(r'^/api/v1/users$'), 'create_user'),
|
||||
('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'),
|
||||
('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'),
|
||||
('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'),
|
||||
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)$'), 'get_chat'),
|
||||
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'list_messages'),
|
||||
('POST', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'post_message'),
|
||||
('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'),
|
||||
('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'),
|
||||
)
|
||||
|
||||
def __init__(self, config, store: Store, database: Database):
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.database = database
|
||||
|
||||
def dispatch(self, request):
|
||||
"""Return (status, payload). Only ApiError and payloads leave this method."""
|
||||
path_matched = False
|
||||
for method, pattern, handler in self.ROUTES:
|
||||
match = pattern.match(request.path)
|
||||
if match:
|
||||
path_matched = True
|
||||
if method == request.method:
|
||||
return getattr(self, handler)(request, match.groupdict())
|
||||
raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается')
|
||||
|
||||
# ------------------------------------------------------------- health
|
||||
def health_live(self, request, path_params):
|
||||
return 200, {'status': 'ok'}
|
||||
|
||||
def health_ready(self, request, path_params):
|
||||
try:
|
||||
self.store.stats()
|
||||
except Exception:
|
||||
raise Unavailable('База данных недоступна')
|
||||
return 200, {'status': 'ok'}
|
||||
|
||||
# -------------------------------------------------------------- users
|
||||
def create_user(self, request, path_params):
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('username', 'display_name', 'password'),
|
||||
('username', 'display_name', 'password'))
|
||||
username = validation.require_username(payload)
|
||||
display_name = validation.require_text_field(
|
||||
payload, 'display_name', 1, self.config.max_display_name_code_points)
|
||||
password = validation.require_text_field(
|
||||
payload, 'password', 12, 128, strip_check=False)
|
||||
password_hash = security.hash_password(password)
|
||||
return 201, self.store.create_user(username, display_name, password_hash)
|
||||
|
||||
def current_user(self, request, path_params):
|
||||
return 200, self._authenticate(request)
|
||||
|
||||
# -------------------------------------------------------------- chats
|
||||
def create_chat(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids'))
|
||||
title = validation.require_text_field(payload, 'title', 1,
|
||||
self.config.max_title_code_points)
|
||||
members = validation.require_member_ids(payload, 100)
|
||||
if actor['id'] not in members:
|
||||
raise InvalidRequest('Создатель чата должен быть указан в member_ids')
|
||||
if not self.store.users_exist(members):
|
||||
raise InvalidRequest('member_ids содержит неизвестных пользователей')
|
||||
return 201, self.store.create_chat(title, members)
|
||||
|
||||
def list_chats(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
limit, after_key = self._pagination(request, SCOPE_CHATS)
|
||||
items, has_more = self.store.page_chats(actor['id'], after_key, limit)
|
||||
for item in items:
|
||||
item['member_ids'] = self._chat_member_ids(item['id'])
|
||||
return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)}
|
||||
|
||||
def get_chat(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
chat['member_ids'] = self._chat_member_ids(chat['id'])
|
||||
return 200, chat
|
||||
|
||||
# ----------------------------------------------------------- messages
|
||||
def post_message(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('text',), ('text',))
|
||||
text = validation.require_text_field(payload, 'text', 1,
|
||||
self.config.max_text_code_points)
|
||||
return 201, self.store.add_message(chat['id'], actor['id'], text)
|
||||
|
||||
def list_messages(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
scope = SCOPE_MESSAGES.format(chat_id=chat['id'])
|
||||
limit, after_key = self._pagination(request, scope)
|
||||
items, has_more = self.store.page_messages(chat['id'], after_key, limit)
|
||||
return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)}
|
||||
|
||||
# ----------------------------------------------------------- sessions
|
||||
def create_session(self, request, path_params):
|
||||
"""Login: only Authorization: Basic username:password, no JSON body."""
|
||||
authorization = request.headers.get('Authorization') or ''
|
||||
if not authorization.lower().startswith('basic '):
|
||||
raise Unauthorized('Требуется авторизация Basic')
|
||||
try:
|
||||
raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8')
|
||||
except Exception:
|
||||
raise Unauthorized('Некорректные учётные данные') from None
|
||||
if ':' not in raw:
|
||||
raise Unauthorized('Некорректные учётные данные')
|
||||
username, password = raw.split(':', 1)
|
||||
row = self.store.user_by_username(username)
|
||||
if row is None or not security.verify_password(password, row['password_hash']):
|
||||
raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"')
|
||||
token = security.generate_token()
|
||||
token_hash = security.hash_token(token)
|
||||
expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds)
|
||||
self.store.create_session(token_hash, row['id'], expires_at)
|
||||
user = self.store._public_user({
|
||||
'id': row['id'],
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': row['created_at'],
|
||||
})
|
||||
return 201, {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': rfc3339(expires_at),
|
||||
'user': user,
|
||||
}
|
||||
|
||||
def revoke_session(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
token = self._bearer_token(request)
|
||||
self.store.delete_session(security.hash_token(token))
|
||||
return 204, None
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
def _authenticate(self, request):
|
||||
"""Lab2 identity: only Authorization: Bearer <token> identifies the actor.
|
||||
|
||||
X-User-Id is ignored: it does not change the session owner and alone
|
||||
grants no access. Basic on business endpoints is 401.
|
||||
"""
|
||||
token = self._bearer_token(request)
|
||||
if token is None:
|
||||
raise Unauthorized('Требуется идентификация пользователя',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
user_id = self.store.get_session(security.hash_token(token))
|
||||
if user_id is None:
|
||||
raise Unauthorized('Сессия недействительна или истекла',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
user = self.store.user(user_id)
|
||||
if user is None:
|
||||
raise Unauthorized('Пользователь не найден',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
return user
|
||||
|
||||
def _bearer_token(self, request):
|
||||
authorization = request.headers.get('Authorization') or ''
|
||||
if not authorization.lower().startswith('bearer '):
|
||||
return None
|
||||
return authorization[len('bearer '):].strip()
|
||||
|
||||
def _chat_member_ids(self, chat_id):
|
||||
with self.database.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
|
||||
return [str(row['user_id']) for row in cur.fetchall()]
|
||||
|
||||
def _member_chat(self, actor, raw_chat_id):
|
||||
"""Foreign and missing chats are indistinguishable: both are 404."""
|
||||
chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id')
|
||||
chat = self.store.chat(chat_id)
|
||||
if chat is None or not self.store.is_member(chat, actor['id']):
|
||||
raise NotFound('Чат не найден')
|
||||
return chat
|
||||
|
||||
def _json_object(self, request):
|
||||
content_type = request.headers.get('Content-Type') or ''
|
||||
if content_type.split(';')[0].strip().lower() != 'application/json':
|
||||
raise UnsupportedMediaType('Ожидается Content-Type: application/json')
|
||||
if not request.body:
|
||||
raise InvalidRequest('Требуется JSON-тело запроса')
|
||||
try:
|
||||
payload = json.loads(request.body.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
raise InvalidRequest('Тело запроса не является корректным JSON') from None
|
||||
return validation.require_object(payload)
|
||||
|
||||
def _pagination(self, request, scope):
|
||||
raw_limit = (request.query.get('limit') or [None])[0]
|
||||
limit = validation.parse_limit(raw_limit, self.config.default_page_limit,
|
||||
self.config.max_page_limit)
|
||||
raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0])
|
||||
if raw_cursor is None:
|
||||
return limit, None
|
||||
try:
|
||||
stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret)
|
||||
except cursor_codec.InvalidCursor:
|
||||
raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None
|
||||
return limit, (stamp, identifier)
|
||||
|
||||
def _next_cursor(self, scope, items, has_more):
|
||||
if not has_more or not items:
|
||||
return None
|
||||
last = items[-1]
|
||||
return cursor_codec.encode(scope, last['created_at'], last['id'],
|
||||
self.config.cursor_secret)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Redis session cache with TTL and graceful degradation.
|
||||
|
||||
PostgreSQL is the source of truth for sessions. Redis is a look-aside cache:
|
||||
- write: session is persisted in PostgreSQL first, then copied to Redis with TTL
|
||||
- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis
|
||||
- when Redis is unavailable: fall back to PostgreSQL only, never treat an
|
||||
anonymous request as trusted
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import redis
|
||||
|
||||
from .config import Config
|
||||
|
||||
logger = logging.getLogger('messenger.cache')
|
||||
|
||||
|
||||
class SessionCache:
|
||||
"""Look-aside cache for session tokens in Redis/Valkey."""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
self._client = None
|
||||
self._available = False
|
||||
|
||||
def open(self):
|
||||
if not self._config.cache_url:
|
||||
self._available = False
|
||||
return
|
||||
try:
|
||||
self._client = redis.Redis.from_url(
|
||||
self._config.cache_url,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2,
|
||||
socket_keepalive=True,
|
||||
health_check_interval=30,
|
||||
decode_responses=True,
|
||||
)
|
||||
self._client.ping()
|
||||
self._available = True
|
||||
logger.info('cache_connected', extra={'event': 'cache_connected'})
|
||||
except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely
|
||||
self._available = False
|
||||
self._client = None
|
||||
logger.warning('cache_unavailable', extra={
|
||||
'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'})
|
||||
|
||||
def close(self):
|
||||
if self._client is not None:
|
||||
try:
|
||||
self._client.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._client = None
|
||||
self._available = False
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
return self._available and self._client is not None
|
||||
|
||||
def get(self, token_hash):
|
||||
"""Return cached user_id or None (miss or unavailable)."""
|
||||
if not self.available:
|
||||
return None
|
||||
try:
|
||||
value = self._client.get('session:%s' % token_hash)
|
||||
return value
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_read_failed', extra={
|
||||
'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'})
|
||||
self._available = False
|
||||
return None
|
||||
|
||||
def set(self, token_hash, user_id, ttl_seconds):
|
||||
"""Cache a session with TTL. Failure is non-fatal."""
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self._client.setex('session:%s' % token_hash, ttl_seconds, user_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_write_failed', extra={
|
||||
'event': 'cache_write_failed', 'detail': 'continuing without cache'})
|
||||
self._available = False
|
||||
|
||||
def delete(self, token_hash):
|
||||
"""Remove a session from the cache (logout). Failure is non-fatal."""
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self._client.delete('session:%s' % token_hash)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_delete_failed', extra={
|
||||
'event': 'cache_delete_failed'})
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Runtime configuration. Every value is read from the environment."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
|
||||
# Contract limits: they cannot be raised by configuration.
|
||||
CONTRACT_MAX_TEXT_CODE_POINTS = 4000
|
||||
CONTRACT_MAX_PAGE_LIMIT = 100
|
||||
CONTRACT_SESSION_TTL_MIN = 1
|
||||
CONTRACT_SESSION_TTL_MAX = 86400
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Raised when an environment value cannot be used."""
|
||||
|
||||
|
||||
def _str(environ, name, default):
|
||||
value = environ.get(name)
|
||||
if value is None or value == '':
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _int(environ, name, default, minimum, maximum):
|
||||
raw = environ.get(name)
|
||||
if raw is None or raw == '':
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None
|
||||
if not minimum <= value <= maximum:
|
||||
raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value))
|
||||
return value
|
||||
|
||||
|
||||
def _log_level(environ, name, default):
|
||||
value = _str(environ, name, default).upper()
|
||||
if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
|
||||
raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name)
|
||||
return value
|
||||
|
||||
|
||||
def _secret_from_file(environ, name, default=None):
|
||||
"""Читает секрет из файла, путь к которому задаёт переменная окружения.
|
||||
|
||||
Секреты не лежат в переменных окружения напрямую: контейнер монтирует
|
||||
файл (например, Docker secret), а приложение читает его содержимое.
|
||||
Если файл не задан или не существует, используется значение переменной
|
||||
окружения без суффикса _FILE (удобно для локального запуска без секретов).
|
||||
"""
|
||||
path_name = name + '_FILE'
|
||||
path = environ.get(path_name)
|
||||
if path:
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as handle:
|
||||
return handle.read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
return environ.get(name) or default
|
||||
|
||||
|
||||
class Config:
|
||||
"""Effective configuration of one process instance."""
|
||||
|
||||
def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds,
|
||||
max_text_code_points, max_display_name_code_points, max_title_code_points,
|
||||
default_page_limit, max_page_limit, max_body_bytes,
|
||||
database_url, cache_url, session_ttl_seconds, session_store,
|
||||
pool_min, pool_max, connect_timeout, statement_timeout):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.env = env
|
||||
self.log_level = log_level
|
||||
self.cursor_secret = cursor_secret
|
||||
self.shutdown_grace_seconds = shutdown_grace_seconds
|
||||
self.max_text_code_points = max_text_code_points
|
||||
self.max_display_name_code_points = max_display_name_code_points
|
||||
self.max_title_code_points = max_title_code_points
|
||||
self.default_page_limit = default_page_limit
|
||||
self.max_page_limit = max_page_limit
|
||||
self.max_body_bytes = max_body_bytes
|
||||
self.database_url = database_url
|
||||
self.cache_url = cache_url
|
||||
self.session_ttl_seconds = session_ttl_seconds
|
||||
self.session_store = session_store
|
||||
self.pool_min = pool_min
|
||||
self.pool_max = pool_max
|
||||
self.connect_timeout = connect_timeout
|
||||
self.statement_timeout = statement_timeout
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, environ=None):
|
||||
environ = os.environ if environ is None else environ
|
||||
secret = environ.get('CURSOR_SECRET') or ''
|
||||
if not secret:
|
||||
secret = secrets.token_hex(32)
|
||||
session_store = _str(environ, 'SESSION_STORE', 'postgres')
|
||||
if session_store not in ('postgres', 'redis'):
|
||||
raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store)
|
||||
return cls(
|
||||
host=_str(environ, 'APP_HOST', '0.0.0.0'),
|
||||
port=_int(environ, 'APP_PORT', 8080, 1, 65535),
|
||||
env=_str(environ, 'APP_ENV', 'lab'),
|
||||
log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'),
|
||||
cursor_secret=secret.encode('utf-8'),
|
||||
shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10),
|
||||
max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS,
|
||||
1, CONTRACT_MAX_TEXT_CODE_POINTS),
|
||||
max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100),
|
||||
max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100),
|
||||
default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT),
|
||||
max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1,
|
||||
CONTRACT_MAX_PAGE_LIMIT),
|
||||
max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576),
|
||||
database_url=_secret_from_file(environ, 'DATABASE_URL', None),
|
||||
cache_url=_secret_from_file(environ, 'CACHE_URL', None),
|
||||
session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600,
|
||||
CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX),
|
||||
session_store=session_store,
|
||||
pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50),
|
||||
pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100),
|
||||
connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60),
|
||||
statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300),
|
||||
)
|
||||
|
||||
def as_log_fields(self):
|
||||
"""Configuration summary safe for logs (no secrets)."""
|
||||
return {
|
||||
'app_env': self.env,
|
||||
'host': self.host,
|
||||
'port': self.port,
|
||||
'log_level': self.log_level,
|
||||
'shutdown_grace_seconds': self.shutdown_grace_seconds,
|
||||
'default_page_limit': self.default_page_limit,
|
||||
'max_page_limit': self.max_page_limit,
|
||||
'max_text_code_points': self.max_text_code_points,
|
||||
'max_body_bytes': self.max_body_bytes,
|
||||
'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')),
|
||||
'database_url_configured': bool(self.database_url),
|
||||
'cache_url_configured': bool(self.cache_url),
|
||||
'session_ttl_seconds': self.session_ttl_seconds,
|
||||
'session_store': self.session_store,
|
||||
'db_pool_min': self.pool_min,
|
||||
'db_pool_max': self.pool_max,
|
||||
'db_connect_timeout': self.connect_timeout,
|
||||
'db_statement_timeout': self.statement_timeout,
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import uuid
|
||||
|
||||
VERSION = 1
|
||||
MAC_BYTES = 16
|
||||
|
||||
|
||||
class InvalidCursor(ValueError):
|
||||
"""Cursor is malformed, forged, expired or belongs to another resource."""
|
||||
|
||||
|
||||
def _key(secret):
|
||||
return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest()
|
||||
|
||||
|
||||
def encode(scope, created_at, object_id, secret):
|
||||
"""Serialize a keyset position bound to one resource scope."""
|
||||
payload = json.dumps(
|
||||
{'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)},
|
||||
separators=(',', ':'), sort_keys=True).encode('utf-8')
|
||||
mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]
|
||||
return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode(token, scope, secret):
|
||||
"""Return (created_at datetime, uuid) or raise InvalidCursor."""
|
||||
if not isinstance(token, str) or not token:
|
||||
raise InvalidCursor('empty cursor')
|
||||
padding = '=' * (-len(token) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(token + padding)
|
||||
except (binascii.Error, ValueError):
|
||||
raise InvalidCursor('cursor is not base64url') from None
|
||||
if len(raw) <= MAC_BYTES:
|
||||
raise InvalidCursor('cursor is too short')
|
||||
payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:]
|
||||
if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]):
|
||||
raise InvalidCursor('cursor signature mismatch')
|
||||
try:
|
||||
data = json.loads(payload.decode('utf-8'))
|
||||
except (ValueError, UnicodeError):
|
||||
raise InvalidCursor('cursor payload is not JSON') from None
|
||||
if not isinstance(data, dict) or data.get('v') != VERSION:
|
||||
raise InvalidCursor('unsupported cursor version')
|
||||
if data.get('scope') != scope:
|
||||
raise InvalidCursor('cursor belongs to another resource')
|
||||
created_at = data.get('created_at')
|
||||
object_id = data.get('id')
|
||||
if not isinstance(created_at, str) or not isinstance(object_id, str):
|
||||
raise InvalidCursor('cursor payload types')
|
||||
try:
|
||||
stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00'))
|
||||
identifier = uuid.UUID(object_id)
|
||||
except ValueError:
|
||||
raise InvalidCursor('cursor payload values') from None
|
||||
if stamp.tzinfo is None:
|
||||
raise InvalidCursor('cursor timestamp has no timezone')
|
||||
return stamp.astimezone(dt.timezone.utc), identifier
|
||||
@@ -0,0 +1,83 @@
|
||||
"""PostgreSQL connection pool and schema migrations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg2 import pool
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
from .config import Config
|
||||
from .errors import Unavailable
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MIGRATIONS_DIR = _ROOT / 'migrations'
|
||||
|
||||
|
||||
class Database:
|
||||
"""Thread-safe connection pool with prepared statement timeouts.
|
||||
|
||||
All queries are parameterised (%s). Connections are returned to the pool
|
||||
after every operation; the pool itself is closed on shutdown.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
self._pool = None
|
||||
|
||||
def open(self):
|
||||
if self._pool is not None:
|
||||
return
|
||||
try:
|
||||
self._pool = pool.ThreadedConnectionPool(
|
||||
minconn=self._config.pool_min,
|
||||
maxconn=self._config.pool_max,
|
||||
dsn=self._config.database_url,
|
||||
connect_timeout=self._config.connect_timeout,
|
||||
cursor_factory=RealDictCursor,
|
||||
)
|
||||
except Exception as exc: # psycopg2.Error or configuration error
|
||||
raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None
|
||||
|
||||
def close(self):
|
||||
if self._pool is not None:
|
||||
self._pool.closeall()
|
||||
self._pool = None
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
"""Yield a connection; rollback on exception, return to pool on exit."""
|
||||
if self._pool is None:
|
||||
raise Unavailable('База данных не инициализирована')
|
||||
conn = self._pool.getconn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('SET statement_timeout = %s',
|
||||
(self._config.statement_timeout * 1000,))
|
||||
yield conn
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
self._pool.putconn(conn)
|
||||
|
||||
@contextmanager
|
||||
def cursor(self):
|
||||
"""Yield a cursor inside a connection; commit on success, rollback on failure."""
|
||||
with self.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
yield cur
|
||||
conn.commit()
|
||||
|
||||
def run_migrations(self):
|
||||
"""Apply every .sql migration file in order. Idempotent (IF NOT EXISTS)."""
|
||||
files = sorted(p for p in _MIGRATIONS_DIR.iterdir()
|
||||
if p.suffix == '.sql' and p.is_file())
|
||||
for path in files:
|
||||
sql = path.read_text(encoding='utf-8')
|
||||
with self.cursor() as cur:
|
||||
cur.execute(sql)
|
||||
@@ -0,0 +1,73 @@
|
||||
"""API error taxonomy; every message is safe to return to a client."""
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""Base class for errors that map onto the public error payload."""
|
||||
|
||||
status = 500
|
||||
code = 'internal_error'
|
||||
default_message = 'Внутренняя ошибка сервера'
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.default_message
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class InvalidRequest(ApiError):
|
||||
status = 400
|
||||
code = 'invalid_request'
|
||||
default_message = 'Некорректный запрос'
|
||||
|
||||
|
||||
class Unauthorized(ApiError):
|
||||
status = 401
|
||||
code = 'unauthorized'
|
||||
default_message = 'Требуется идентификация пользователя'
|
||||
www_authenticate = None
|
||||
|
||||
def __init__(self, message=None, www_authenticate=None):
|
||||
self.message = message or self.default_message
|
||||
self.www_authenticate = www_authenticate
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class NotFound(ApiError):
|
||||
status = 404
|
||||
code = 'not_found'
|
||||
default_message = 'Объект не найден'
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
status = 409
|
||||
code = 'conflict'
|
||||
default_message = 'Конфликт состояния'
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
status = 413
|
||||
code = 'payload_too_large'
|
||||
default_message = 'Тело запроса слишком велико'
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
status = 415
|
||||
code = 'unsupported_media_type'
|
||||
default_message = 'Поддерживается только application/json'
|
||||
|
||||
|
||||
class RateLimited(ApiError):
|
||||
status = 429
|
||||
code = 'rate_limited'
|
||||
default_message = 'Слишком много запросов'
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
status = 503
|
||||
code = 'unavailable'
|
||||
default_message = 'Сервис временно недоступен'
|
||||
|
||||
|
||||
class InternalError(ApiError):
|
||||
status = 500
|
||||
code = 'internal_error'
|
||||
default_message = 'Внутренняя ошибка сервера'
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Structured stdout logging without payload contents."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id',
|
||||
'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version')
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line; message text of API payloads is never logged."""
|
||||
|
||||
def format(self, record):
|
||||
payload = {
|
||||
'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) +
|
||||
'.%03dZ' % int(record.msecs),
|
||||
'level': record.levelname.lower(),
|
||||
'logger': record.name,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
for field in _FIELDS:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None:
|
||||
payload[field] = value
|
||||
if record.exc_info and record.levelno >= logging.ERROR:
|
||||
payload['exception'] = record.exc_info[0].__name__
|
||||
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def configure(level_name):
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
for existing in list(root.handlers):
|
||||
root.removeHandler(existing)
|
||||
root.addHandler(handler)
|
||||
root.setLevel(getattr(logging, level_name))
|
||||
return logging.getLogger('messenger')
|
||||
|
||||
|
||||
def access_logger():
|
||||
return logging.getLogger('messenger.access')
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Password hashing and session tokens.
|
||||
|
||||
Passwords are hashed with a library password KDF: scrypt from the standard
|
||||
library when available, otherwise bcrypt with a SHA-256 pre-hash so that
|
||||
passwords longer than bcrypt's 72-byte limit are still supported.
|
||||
Session tokens are opaque random values (>= 256 bits of entropy); only their
|
||||
SHA-256 is stored in the database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
try: # scrypt is part of hashlib on OpenSSL-enabled builds
|
||||
hashlib.scrypt(b'x', b's', n=16, r=8, p=1, dklen=16)
|
||||
_HAS_SCRYPT = True
|
||||
except Exception:
|
||||
_HAS_SCRYPT = False
|
||||
|
||||
if not _HAS_SCRYPT:
|
||||
import bcrypt # type: ignore
|
||||
|
||||
SCRYPT_N = 16384
|
||||
SCRYPT_R = 8
|
||||
SCRYPT_P = 1
|
||||
SCRYPT_DKLEN = 32
|
||||
SALT_BYTES = 16
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Hash a UTF-8 password with a library password KDF.
|
||||
|
||||
Returns a single storable string:
|
||||
scrypt:$n:$r:$p:$salt_b64:$hash_b64
|
||||
or
|
||||
bcrypt:$salt_b64:$hash_b64
|
||||
"""
|
||||
raw = password.encode('utf-8')
|
||||
if _HAS_SCRYPT:
|
||||
salt = secrets.token_bytes(SALT_BYTES)
|
||||
dk = hashlib.scrypt(
|
||||
raw,
|
||||
salt=salt,
|
||||
n=SCRYPT_N,
|
||||
r=SCRYPT_R,
|
||||
p=SCRYPT_P,
|
||||
dklen=SCRYPT_DKLEN,
|
||||
)
|
||||
return 'scrypt:%d:%d:%d:%s:%s' % (
|
||||
SCRYPT_N, SCRYPT_R, SCRYPT_P,
|
||||
base64.b64encode(salt).decode('ascii'),
|
||||
base64.b64encode(dk).decode('ascii'),
|
||||
)
|
||||
# bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords
|
||||
# (up to 128 code points per contract) are fully covered.
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12))
|
||||
return 'bcrypt:%s' % hashed.decode('ascii')
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
"""Verify a password against a KDF hash string. Constant-time compare."""
|
||||
try:
|
||||
raw = password.encode('utf-8')
|
||||
if password_hash.startswith('scrypt:'):
|
||||
parts = password_hash.split('$')
|
||||
if len(parts) != 6 or parts[0] != 'scrypt':
|
||||
return False
|
||||
_, n, r, p, salt_b64, hash_b64 = parts
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(hash_b64)
|
||||
dk = hashlib.scrypt(
|
||||
raw,
|
||||
salt=salt,
|
||||
n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(expected),
|
||||
)
|
||||
return hmac.compare_digest(dk, expected)
|
||||
if password_hash.startswith('bcrypt:'):
|
||||
stored = password_hash[len('bcrypt:'):].encode('ascii')
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
return bcrypt.checkpw(digest, stored)
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def generate_token() -> str:
|
||||
"""Opaque random bearer token, at least 256 bits of entropy."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
"""SHA-256 of a bearer token, stored in the database instead of the token itself."""
|
||||
return hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
@@ -0,0 +1,259 @@
|
||||
"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown."""
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from . import __version__, logging_utils
|
||||
from .api import Api, Request
|
||||
from .cache import SessionCache
|
||||
from .config import Config, ConfigError
|
||||
from .db import Database
|
||||
from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable
|
||||
from .store import Store
|
||||
|
||||
MAX_REQUEST_ID_LENGTH = 128
|
||||
MAX_LOGGED_LINE = 200
|
||||
|
||||
|
||||
def sanitize_request_id(raw):
|
||||
"""Echo a safe client id, otherwise generate one. Never trust raw input."""
|
||||
if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw):
|
||||
return raw
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def route_template(path):
|
||||
for _method, pattern, _handler in Api.ROUTES:
|
||||
if pattern.match(path):
|
||||
return pattern.pattern.replace('[^/]+', '{chat_id}').strip('^$')
|
||||
return 'unmatched'
|
||||
|
||||
|
||||
def encode_payload(payload):
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||
|
||||
|
||||
class ApiServer(ThreadingHTTPServer):
|
||||
"""Threaded server that tracks in-flight requests and stops accepting on demand."""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address, api, config, logger, database):
|
||||
self.api = api
|
||||
self.config = config
|
||||
self.logger = logger
|
||||
self.database = database
|
||||
self.shutting_down = False
|
||||
self._active = 0
|
||||
self._idle = threading.Condition()
|
||||
super().__init__(address, ApiHandler)
|
||||
|
||||
def enter(self):
|
||||
with self._idle:
|
||||
self._active += 1
|
||||
|
||||
def leave(self):
|
||||
with self._idle:
|
||||
self._active -= 1
|
||||
if self._active == 0:
|
||||
self._idle.notify_all()
|
||||
|
||||
def wait_idle(self, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
with self._idle:
|
||||
while self._active > 0:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
self._idle.wait(remaining)
|
||||
return True
|
||||
|
||||
|
||||
class ApiHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
server_version = 'mtusi-messenger/' + __version__
|
||||
sys_version = ''
|
||||
timeout = 30
|
||||
|
||||
# --------------------------------------------------------- entry points
|
||||
def do_GET(self):
|
||||
self._handle()
|
||||
|
||||
do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
"""Base-class chatter stays debug-only; access logs are structured."""
|
||||
self.server.logger.debug('http_note', extra={
|
||||
'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]})
|
||||
|
||||
# ------------------------------------------------------------- pipeline
|
||||
def _handle(self):
|
||||
started = time.monotonic()
|
||||
server = self.server
|
||||
request_id = sanitize_request_id(self.headers.get('X-Request-Id'))
|
||||
head_only = self.command == 'HEAD'
|
||||
path = urlsplit(self.path).path
|
||||
status, error_code = 500, None
|
||||
self._last_exc = None
|
||||
server.enter()
|
||||
try:
|
||||
if server.shutting_down:
|
||||
raise Unavailable('Сервер завершает работу')
|
||||
body = self._read_body()
|
||||
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
|
||||
request = Request('GET' if head_only else self.command, path, query, self.headers,
|
||||
body, request_id, self.client_address[0])
|
||||
status, payload = server.api.dispatch(request)
|
||||
encoded = encode_payload(payload)
|
||||
except ApiError as exc:
|
||||
self._last_exc = exc
|
||||
status, error_code = exc.status, exc.code
|
||||
encoded = self._error_payload(exc, request_id)
|
||||
except Exception: # last resort: log privately, answer generically
|
||||
server.logger.exception('unhandled_error', extra={
|
||||
'event': 'unhandled_error', 'request_id': request_id,
|
||||
'method': self.command, 'route': route_template(path)})
|
||||
status, error_code = 500, 'internal_error'
|
||||
encoded = self._error_payload(ApiError(), request_id)
|
||||
finally:
|
||||
server.leave()
|
||||
server.logger.info('http_request', extra={
|
||||
'event': 'http_request', 'request_id': request_id, 'method': self.command,
|
||||
'route': route_template(path), 'status': status,
|
||||
'duration_ms': round(1000 * (time.monotonic() - started), 3),
|
||||
'error_code': error_code, 'client': self.client_address[0]})
|
||||
self._send(status, encoded, request_id, head_only)
|
||||
|
||||
def _read_body(self):
|
||||
if self.headers.get('Transfer-Encoding'):
|
||||
self.close_connection = True
|
||||
raise InvalidRequest('Transfer-Encoding не поддерживается')
|
||||
raw_length = self.headers.get('Content-Length')
|
||||
if raw_length is None:
|
||||
return b''
|
||||
try:
|
||||
length = int(raw_length.strip())
|
||||
except ValueError:
|
||||
raise InvalidRequest('Некорректный Content-Length') from None
|
||||
if length < 0:
|
||||
raise InvalidRequest('Некорректный Content-Length')
|
||||
if length > self.server.config.max_body_bytes:
|
||||
self.close_connection = True
|
||||
raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes)
|
||||
if length == 0:
|
||||
return b''
|
||||
return self.rfile.read(length)
|
||||
|
||||
@staticmethod
|
||||
def _error_payload(exc, request_id):
|
||||
return encode_payload({'error': {
|
||||
'code': exc.code, 'message': exc.message, 'request_id': request_id}})
|
||||
|
||||
def _send(self, status, encoded, request_id, head_only):
|
||||
try:
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
||||
self.send_header('X-Request-Id', request_id)
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
if status == 204:
|
||||
self.send_header('Content-Length', '0')
|
||||
if status == 503:
|
||||
self.send_header('Retry-After', '1')
|
||||
self.end_headers()
|
||||
return
|
||||
if status == 503:
|
||||
self.send_header('Retry-After', '1')
|
||||
if status == 401 and self._last_exc is not None:
|
||||
www_auth = getattr(self._last_exc, 'www_authenticate', None)
|
||||
if www_auth:
|
||||
self.send_header('WWW-Authenticate', www_auth)
|
||||
if self.close_connection:
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
if not head_only and encoded:
|
||||
self.wfile.write(encoded)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
def build_server(config, logger, api, database):
|
||||
"""Create the listening server; raises OSError when the port is taken."""
|
||||
return ApiServer((config.host, config.port), api, config, logger, database)
|
||||
|
||||
|
||||
def run(config, logger, database, cache):
|
||||
"""Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period."""
|
||||
api = Api(config, Store(config, database, cache), database)
|
||||
try:
|
||||
server = build_server(config, logger, api, database)
|
||||
except OSError as exc:
|
||||
logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)})
|
||||
return 1
|
||||
stop = threading.Event()
|
||||
|
||||
def handle_signal(signum, _frame):
|
||||
if stop.is_set():
|
||||
return
|
||||
stop.set()
|
||||
server.shutting_down = True
|
||||
logger.warning('shutdown_signal', extra={
|
||||
'event': 'shutdown_signal', 'signal': signum,
|
||||
'detail': 'stop accepting new requests'})
|
||||
threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start()
|
||||
|
||||
for name in ('SIGTERM', 'SIGINT'):
|
||||
if hasattr(signal, name):
|
||||
signal.signal(getattr(signal, name), handle_signal)
|
||||
|
||||
logger.info('server_started', extra=dict(
|
||||
{'event': 'server_started', 'version': __version__,
|
||||
'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())},
|
||||
**config.as_log_fields()))
|
||||
started = time.monotonic()
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
finally:
|
||||
server.server_close()
|
||||
drained = server.wait_idle(config.shutdown_grace_seconds)
|
||||
database.close()
|
||||
logger.info('server_stopped', extra={
|
||||
'event': 'server_stopped',
|
||||
'outcome': 'graceful' if drained else 'grace_period_exceeded',
|
||||
'duration_ms': round(1000 * (time.monotonic() - started), 3)})
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
config = Config.from_env()
|
||||
except ConfigError as exc:
|
||||
print('configuration error: %s' % exc, file=sys.stderr)
|
||||
return 2
|
||||
logger = logging_utils.configure(config.log_level)
|
||||
database = Database(config)
|
||||
cache = SessionCache(config)
|
||||
try:
|
||||
database.open()
|
||||
database.run_migrations()
|
||||
cache.open()
|
||||
except Exception as exc: # noqa: BLE001 - startup must fail loud and clear
|
||||
logger.error('database_start_failed', extra={
|
||||
'event': 'database_start_failed', 'detail': str(exc)})
|
||||
database.close()
|
||||
cache.close()
|
||||
return 1
|
||||
try:
|
||||
return run(config, logger, database, cache)
|
||||
finally:
|
||||
cache.close()
|
||||
@@ -0,0 +1,266 @@
|
||||
"""PostgreSQL-backed state store. All writes go through parameterised queries
|
||||
inside explicit transactions; reads use keyset pagination over indexed columns."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import uuid
|
||||
|
||||
from . import cursor as cursor_codec
|
||||
from . import security
|
||||
from .cache import SessionCache
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import Conflict, InvalidRequest, NotFound
|
||||
|
||||
|
||||
def utc_now():
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def rfc3339(stamp):
|
||||
"""RFC 3339 UTC with fixed microsecond precision and the Z suffix."""
|
||||
return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL state: users, chats, membership, messages, sessions."""
|
||||
|
||||
def __init__(self, config: Config, database: Database, cache: SessionCache):
|
||||
self._config = config
|
||||
self._db = database
|
||||
self._cache = cache
|
||||
|
||||
# ---------------------------------------------------------------- reads
|
||||
def stats(self):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('SELECT (SELECT count(*) FROM users) AS users,'
|
||||
' (SELECT count(*) FROM chats) AS chats,'
|
||||
' (SELECT count(*) FROM messages) AS messages')
|
||||
row = cur.fetchone()
|
||||
return {
|
||||
'users': row['users'],
|
||||
'chats': row['chats'],
|
||||
'messages': row['messages'],
|
||||
}
|
||||
|
||||
def user(self, user_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
|
||||
(str(user_id),))
|
||||
row = cur.fetchone()
|
||||
return self._public_user(row) if row else None
|
||||
|
||||
def user_by_username(self, username):
|
||||
"""Return user row including password_hash, or None."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, username, password_hash, display_name, created_at FROM users '
|
||||
'WHERE username = %s', (username,))
|
||||
return cur.fetchone()
|
||||
|
||||
def users_exist(self, identifiers):
|
||||
if not identifiers:
|
||||
return True
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])',
|
||||
([str(i) for i in identifiers],))
|
||||
row = cur.fetchone()
|
||||
return row['c'] == len(identifiers)
|
||||
|
||||
# --------------------------------------------------------------- writes
|
||||
def create_user(self, username, display_name, password_hash):
|
||||
with self._db.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
'INSERT INTO users (id, username, password_hash, display_name, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s) '
|
||||
'RETURNING id, username, display_name, created_at',
|
||||
(str(uuid.uuid4()), username, password_hash, display_name, utc_now()))
|
||||
except Exception:
|
||||
# Unique violation on username -> conflict, rolled back by cursor context.
|
||||
raise Conflict('username уже занят') from None
|
||||
row = cur.fetchone()
|
||||
return self._public_user(row)
|
||||
|
||||
def create_chat(self, title, member_ids):
|
||||
"""Atomically create a chat and its membership. Rollback on any failure."""
|
||||
identifier = str(uuid.uuid4())
|
||||
stamp = utc_now()
|
||||
with self._db.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(identifier, title, stamp))
|
||||
for member_id in member_ids:
|
||||
cur.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)',
|
||||
(identifier, str(member_id)))
|
||||
except Exception:
|
||||
raise InvalidRequest('member_ids содержит неизвестных пользователей') from None
|
||||
return {
|
||||
'id': identifier,
|
||||
'title': title,
|
||||
'member_ids': list(member_ids),
|
||||
'created_at': rfc3339(stamp),
|
||||
}
|
||||
|
||||
def add_message(self, chat_id, sender_id, text):
|
||||
identifier = str(uuid.uuid4())
|
||||
stamp = utc_now()
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s) '
|
||||
'RETURNING id, chat_id, sender_id, text, created_at',
|
||||
(identifier, str(chat_id), str(sender_id), text, stamp))
|
||||
row = cur.fetchone()
|
||||
return self._public_message(row)
|
||||
|
||||
# --------------------------------------------------------------- pages
|
||||
def page_messages(self, chat_id, after_key, limit):
|
||||
"""Return (public messages, has_more) for a keyset window."""
|
||||
with self._db.cursor() as cur:
|
||||
if after_key is None:
|
||||
cur.execute(
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
'ORDER BY created_at, id '
|
||||
'LIMIT %s', (str(chat_id), limit + 1))
|
||||
else:
|
||||
stamp, identifier = after_key
|
||||
cur.execute(
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
'AND (created_at, id) > (%s, %s) '
|
||||
'ORDER BY created_at, id '
|
||||
'LIMIT %s',
|
||||
(str(chat_id), stamp, str(identifier), limit + 1))
|
||||
rows = cur.fetchall()
|
||||
has_more = len(rows) > limit
|
||||
items = [self._public_message(dict(row)) for row in rows[:limit]]
|
||||
return items, has_more
|
||||
|
||||
def page_chats(self, user_id, after_key, limit):
|
||||
"""Return (chats visible to the user, has_more) ordered by (created_at, id)."""
|
||||
with self._db.cursor() as cur:
|
||||
if after_key is None:
|
||||
cur.execute(
|
||||
'SELECT c.id, c.title, c.created_at FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE cm.user_id = %s '
|
||||
'ORDER BY c.created_at, c.id '
|
||||
'LIMIT %s', (str(user_id), limit + 1))
|
||||
else:
|
||||
stamp, identifier = after_key
|
||||
cur.execute(
|
||||
'SELECT c.id, c.title, c.created_at FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE cm.user_id = %s '
|
||||
'AND (c.created_at, c.id) > (%s, %s) '
|
||||
'ORDER BY c.created_at, c.id '
|
||||
'LIMIT %s',
|
||||
(str(user_id), stamp, str(identifier), limit + 1))
|
||||
rows = cur.fetchall()
|
||||
has_more = len(rows) > limit
|
||||
items = [self._public_chat(dict(row)) for row in rows[:limit]]
|
||||
return items, has_more
|
||||
|
||||
# ----------------------------------------------------------- projection
|
||||
def chat(self, chat_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
chat = self._public_chat(dict(row))
|
||||
cur.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
|
||||
chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()]
|
||||
return chat
|
||||
|
||||
@staticmethod
|
||||
def is_member(chat, user_id):
|
||||
return str(user_id) in chat['member_ids']
|
||||
|
||||
@staticmethod
|
||||
def _public_user(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_chat(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'title': row['title'],
|
||||
'member_ids': [], # filled by API when needed
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_message(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'chat_id': str(row['chat_id']),
|
||||
'sender_id': str(row['sender_id']),
|
||||
'text': row['text'],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------- sessions
|
||||
def create_session(self, token_hash, user_id, expires_at):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) '
|
||||
'VALUES (%s, %s, %s, %s)',
|
||||
(token_hash, str(user_id), expires_at, utc_now()))
|
||||
ttl = max(1, int((expires_at - utc_now()).total_seconds()))
|
||||
self._cache.set(token_hash, str(user_id), ttl)
|
||||
|
||||
def get_session(self, token_hash):
|
||||
"""Return user_id for a valid non-expired session, or None.
|
||||
|
||||
The cache is consulted first, but every cache hit is re-validated
|
||||
against PostgreSQL so that a revoked or expired session is never
|
||||
trusted from a stale Redis key. On miss, read PostgreSQL and
|
||||
backfill the cache. Expired sessions found during lookup are deleted.
|
||||
"""
|
||||
cached = self._cache.get(token_hash)
|
||||
if cached is not None:
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s',
|
||||
(token_hash, utc_now()))
|
||||
if cur.fetchone() is None:
|
||||
self._cache.delete(token_hash)
|
||||
return None
|
||||
return cached
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s',
|
||||
(token_hash,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
if row['expires_at'] <= utc_now():
|
||||
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
return None
|
||||
user_id = str(row['user_id'])
|
||||
ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds()))
|
||||
self._cache.set(token_hash, user_id, ttl)
|
||||
return user_id
|
||||
|
||||
def delete_session(self, token_hash):
|
||||
self._cache.delete(token_hash)
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
|
||||
def cleanup_expired_sessions(self):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),))
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Strict request payload validation; any deviation is 400 invalid_request."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from .errors import InvalidRequest
|
||||
|
||||
USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}')
|
||||
DIGITS_PATTERN = re.compile(r'[0-9]+')
|
||||
|
||||
|
||||
def safe_label(value):
|
||||
"""Render untrusted text for an error message without breaking its encoding.
|
||||
|
||||
Field names come straight from the client JSON, so they may contain the same
|
||||
lone surrogates that the payload itself is rejected for.
|
||||
"""
|
||||
text = value if isinstance(value, str) else repr(value)
|
||||
return text.encode('utf-8', 'replace').decode('utf-8')
|
||||
|
||||
|
||||
def require_encodable(value, name):
|
||||
"""Reject text that cannot survive a UTF-8 round trip.
|
||||
|
||||
`json.loads` happily produces lone surrogates from `\ud800` escapes. Such a
|
||||
string cannot be UTF-8 encoded, so it must be rejected before it is stored;
|
||||
otherwise the write is committed and the response encoding fails afterwards.
|
||||
"""
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise InvalidRequest(
|
||||
'Поле %s содержит недопустимые символы Unicode' % name) from None
|
||||
return value
|
||||
|
||||
|
||||
def require_object(payload):
|
||||
if not isinstance(payload, dict):
|
||||
raise InvalidRequest('Тело запроса должно быть JSON-объектом')
|
||||
return payload
|
||||
|
||||
|
||||
def require_fields(payload, required, allowed):
|
||||
"""Reject missing required fields and any unknown field."""
|
||||
missing = [name for name in required if name not in payload]
|
||||
if missing:
|
||||
raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing)))
|
||||
extra = [name for name in payload if name not in allowed]
|
||||
if extra:
|
||||
raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra)))
|
||||
return payload
|
||||
|
||||
|
||||
def require_text_field(payload, name, minimum, maximum, strip_check=True):
|
||||
value = payload.get(name)
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest('Поле %s должно быть строкой' % name)
|
||||
require_encodable(value, name)
|
||||
if strip_check and not value.strip():
|
||||
raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name)
|
||||
if len(value) < minimum:
|
||||
raise InvalidRequest('Поле %s короче %d символов' % (name, minimum))
|
||||
if len(value) > maximum:
|
||||
raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum))
|
||||
return value
|
||||
|
||||
|
||||
def require_username(payload):
|
||||
username = payload.get('username')
|
||||
# fullmatch, not match: a trailing "$" would also accept a final newline.
|
||||
if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None:
|
||||
raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$')
|
||||
return require_encodable(username, 'username')
|
||||
|
||||
|
||||
def require_uuid_string(value, name):
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest('Поле %s должно быть строкой UUID' % name)
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
raise InvalidRequest('Поле %s не является корректным UUID' % name) from None
|
||||
|
||||
|
||||
def require_member_ids(payload, maximum, minimum=1):
|
||||
value = payload.get('member_ids')
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest('member_ids должен быть массивом UUID')
|
||||
if not minimum <= len(value) <= maximum:
|
||||
raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum))
|
||||
identifiers = [require_uuid_string(item, 'member_ids') for item in value]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise InvalidRequest('member_ids содержит повторяющихся участников')
|
||||
return identifiers
|
||||
|
||||
|
||||
def parse_limit(raw, default, maximum):
|
||||
"""Parse a limit query value strictly: absent means default, anything else must be a plain integer.
|
||||
|
||||
An empty value is a client error rather than an absent parameter, and the
|
||||
digit run is length-bounded before int() because Python refuses to convert
|
||||
strings longer than its integer-string limit, which would surface as a 500.
|
||||
"""
|
||||
if raw is None:
|
||||
return default
|
||||
if not DIGITS_PATTERN.fullmatch(raw):
|
||||
raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum)
|
||||
# int() refuses absurdly long digit strings (ValueError -> 500). Comparing the
|
||||
# significant digits against the maximum first rejects them without converting.
|
||||
significant = raw.lstrip('0') or '0'
|
||||
if len(significant) > len(str(maximum)):
|
||||
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
|
||||
value = int(significant)
|
||||
if not 1 <= value <= maximum:
|
||||
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
|
||||
return value
|
||||
|
||||
|
||||
def parse_cursor(raw):
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, str) or not raw:
|
||||
raise InvalidRequest('cursor должен быть непустой строкой')
|
||||
return raw
|
||||
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrency evidence for level 4: N clients write M messages in parallel.
|
||||
|
||||
Command (see REPORT.md and `make concurrency`):
|
||||
python3 student-tests/concurrency.py --clients 10 --messages-per-client 10 \
|
||||
--output evidence/concurrency.json
|
||||
|
||||
Expectation: every acknowledged message is stored exactly once, the full history
|
||||
contains exactly clients*messages_per_client unique messages and the order over
|
||||
pages is strictly increasing by (created_at, id). No retries are hidden: any
|
||||
failed write is reported and fails the run.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
|
||||
from client import Client, Failure, LAB, arguments, check, timestamp # noqa: E402
|
||||
|
||||
MESSAGES_PATH = '/api/v1/chats/%s/messages'
|
||||
|
||||
|
||||
def build_fixture(base_url, ca_file, clients):
|
||||
"""One user per client plus a single chat everybody belongs to."""
|
||||
master = Client(base_url, ca_file, timeout=15)
|
||||
master.call('GET', '/health/ready', schema='Health')
|
||||
actors = [master.user('conc%d' % index) for index in range(clients)]
|
||||
chat = master.call('POST', '/api/v1/chats', 201, actor=actors[0],
|
||||
body={'title': 'Конкурентная запись',
|
||||
'member_ids': [actor['id'] for actor in actors]},
|
||||
schema='Chat')['json']
|
||||
check(set(chat['member_ids']) == {actor['id'] for actor in actors},
|
||||
'chat membership mismatch')
|
||||
return master, actors, chat
|
||||
|
||||
|
||||
def write_phase(base_url, ca_file, actors, chat, messages_per_client, prefix):
|
||||
"""Each client writes its own slice of messages, all starting at once."""
|
||||
barrier = threading.Barrier(len(actors))
|
||||
|
||||
def worker(index):
|
||||
client = Client(base_url, ca_file, timeout=15)
|
||||
actor = actors[index]
|
||||
barrier.wait()
|
||||
acknowledged, errors = [], []
|
||||
started = time.monotonic()
|
||||
for message_index in range(messages_per_client):
|
||||
text = '%s-%03d-%03d' % (prefix, index, message_index)
|
||||
try:
|
||||
message = client.message(actor, chat, text)['json']
|
||||
acknowledged.append((text, message['id']))
|
||||
except Failure as exc:
|
||||
errors.append('%s: %s' % (text, exc))
|
||||
return {'client_index': index, 'acknowledged': acknowledged, 'errors': errors,
|
||||
'duration_seconds': round(time.monotonic() - started, 3)}
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=len(actors)) as pool:
|
||||
results = list(pool.map(worker, range(len(actors))))
|
||||
return results, round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
def read_whole_history(client, actor, chat):
|
||||
items, cursor, pages = [], None, 0
|
||||
while True:
|
||||
query = '?limit=100' + ('' if cursor is None else '&cursor=' + quote(cursor, safe=''))
|
||||
page = client.call('GET', MESSAGES_PATH % chat['id'] + query, actor=actor,
|
||||
schema='MessagePage')['json']
|
||||
items.extend(page['items'])
|
||||
pages += 1
|
||||
cursor = page['next_cursor']
|
||||
check(pages < 100, 'pagination did not terminate')
|
||||
if cursor is None:
|
||||
return items, pages
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--clients', type=int, default=10)
|
||||
parser.add_argument('--messages-per-client', type=int, default=10)
|
||||
parser.add_argument('--output', default=None,
|
||||
help='Optional JSON report path, no session data inside')
|
||||
|
||||
args = arguments('Concurrent writes from independent HTTP clients', configure)
|
||||
check(1 <= args.clients <= 64, 'clients must be 1..64')
|
||||
check(1 <= args.messages_per_client <= 200, 'messages-per-client must be 1..200')
|
||||
expected = args.clients * args.messages_per_client
|
||||
prefix = 'conc-' + uuid.uuid4().hex[:8]
|
||||
|
||||
master, actors, chat = build_fixture(args.base_url, args.ca_file, args.clients)
|
||||
results, elapsed = write_phase(args.base_url, args.ca_file, actors, chat,
|
||||
args.messages_per_client, prefix)
|
||||
acknowledged = [pair for result in results for pair in result['acknowledged']]
|
||||
errors = [error for result in results for error in result['errors']]
|
||||
history, pages = read_whole_history(master, actors[0], chat)
|
||||
|
||||
stored_texts = [message['text'] for message in history]
|
||||
stored_ids = [message['id'] for message in history]
|
||||
expected_texts = {'%s-%03d-%03d' % (prefix, client, index)
|
||||
for client in range(args.clients)
|
||||
for index in range(args.messages_per_client)}
|
||||
missing = sorted(expected_texts.difference(stored_texts))
|
||||
keys = [(timestamp(message['created_at']), uuid.UUID(message['id']).int)
|
||||
for message in history]
|
||||
ordered_correctly = all(keys[index] < keys[index + 1] for index in range(len(keys) - 1))
|
||||
|
||||
report = {
|
||||
'lab': LAB,
|
||||
'workload': 'closed-loop parallel POST messages from independent clients',
|
||||
'clients': args.clients,
|
||||
'messages_per_client': args.messages_per_client,
|
||||
'expected_messages': expected,
|
||||
'acknowledged_201': len(acknowledged),
|
||||
'stored_in_history': len(history),
|
||||
'unique_stored_ids': len(set(stored_ids)),
|
||||
'unique_stored_texts': len(set(stored_texts)),
|
||||
'duplicates': len(stored_ids) - len(set(stored_ids)),
|
||||
'missing_texts': missing,
|
||||
'failed_writes': len(errors),
|
||||
'write_errors': errors[:10],
|
||||
'history_pages': pages,
|
||||
'history_order_strictly_increasing': ordered_correctly,
|
||||
'writers_elapsed_seconds': elapsed,
|
||||
'successful_rps': round(len(acknowledged) / elapsed, 3) if elapsed else None,
|
||||
'per_client': [{'client_index': result['client_index'],
|
||||
'acknowledged': len(result['acknowledged']),
|
||||
'errors': len(result['errors']),
|
||||
'duration_seconds': result['duration_seconds']}
|
||||
for result in sorted(results, key=lambda item: item['client_index'])],
|
||||
'note': ('Client-observed timings: closed-loop, single Docker host, no retries. '
|
||||
'Latency percentiles are intentionally absent here (see make load).'),
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(text + '\n')
|
||||
print(text)
|
||||
check(not errors, 'some parallel writes failed')
|
||||
check(len(acknowledged) == expected, 'not every write was acknowledged')
|
||||
check(len(history) == expected, 'history holds %d messages instead of %d'
|
||||
% (len(history), expected))
|
||||
check(not missing, 'messages missing from the history')
|
||||
check(len(set(stored_ids)) == expected, 'duplicate message ids in the history')
|
||||
check(ordered_correctly, 'history is not strictly ordered by (created_at, id)')
|
||||
print('PASS: %d parallel messages from %d clients stored exactly once' %
|
||||
(expected, args.clients))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print('FAIL:', exc)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Student-owned lab2 tests: sessions, validation, pagination.
|
||||
|
||||
One command: make student-test
|
||||
Direct call: python3 student-tests/run.py --base-url http://localhost:8080
|
||||
|
||||
The suite complements tests/smoke.py and never weakens it. Every run creates
|
||||
synthetic users, chats and messages only; no public check is imported or patched.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import http.client
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
|
||||
from client import Client, Failure, LAB, arguments, check, timestamp # noqa: E402
|
||||
|
||||
if LAB != 2:
|
||||
print('FAIL: student tests target lab2, but lab.json declares lab%d' % LAB)
|
||||
raise SystemExit(2)
|
||||
|
||||
CHAT_PATH = '/api/v1/chats/%s'
|
||||
MESSAGES_PATH = CHAT_PATH + '/messages'
|
||||
MAX_PAGES = 200
|
||||
|
||||
TESTS = []
|
||||
|
||||
|
||||
def test(func):
|
||||
TESTS.append(func)
|
||||
return func
|
||||
|
||||
|
||||
class Context:
|
||||
"""Shared synthetic fixtures: three users, one chat between two of them."""
|
||||
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
self.alice, self.bob, self.eve, self.chat = client.bootstrap()
|
||||
|
||||
def new_chat(self, title, members):
|
||||
return self.client.call('POST', '/api/v1/chats', 201, actor=members[0],
|
||||
body={'title': title,
|
||||
'member_ids': [member['id'] for member in members]},
|
||||
schema='Chat')['json']
|
||||
|
||||
def send(self, actor, chat, text):
|
||||
return self.client.call('POST', MESSAGES_PATH % chat['id'], 201, actor=actor,
|
||||
body={'text': text}, schema='Message')['json']
|
||||
|
||||
def page(self, actor, chat, limit, cursor=None):
|
||||
query = '?limit=%d' % limit
|
||||
if cursor is not None:
|
||||
query += '&cursor=' + quote(cursor, safe='')
|
||||
return self.client.call('GET', MESSAGES_PATH % chat['id'] + query, actor=actor,
|
||||
schema='MessagePage')['json']
|
||||
|
||||
def traverse(self, actor, chat, limits):
|
||||
"""Walk the whole history, changing the page size between requests."""
|
||||
items, cursor, pages = [], None, 0
|
||||
while True:
|
||||
limit = limits[min(pages, len(limits) - 1)]
|
||||
page = self.page(actor, chat, limit, cursor)
|
||||
items.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
pages += 1
|
||||
check(pages < MAX_PAGES, 'pagination did not terminate')
|
||||
if cursor is None:
|
||||
break
|
||||
return items, pages
|
||||
|
||||
|
||||
def ordered(records):
|
||||
"""Contract order: (created_at, id) ascending, exactly as the client sorts."""
|
||||
return sorted(records, key=lambda item: (timestamp(item['created_at']),
|
||||
uuid.UUID(item['id']).int))
|
||||
|
||||
|
||||
def traversal_order(records):
|
||||
return [item['id'] for item in records]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- health
|
||||
@test
|
||||
def health_and_request_id(ctx):
|
||||
for path in ('/health/live', '/health/ready'):
|
||||
result = ctx.client.call('GET', path, schema='Health')
|
||||
check(result['json'] == {'status': 'ok'},
|
||||
'%s must return exactly {"status":"ok"}' % path)
|
||||
check(len(result['headers']['X-Request-Id']) >= 8, 'request id looks too short')
|
||||
marker = 'student-' + uuid.uuid4().hex
|
||||
echoed = ctx.client.call('GET', '/health/live', headers={'X-Request-Id': marker})
|
||||
check(echoed['headers']['X-Request-Id'] == marker, 'client request id was not echoed')
|
||||
error = ctx.client.call('GET', '/api/v1/users/me', 401, headers={'X-Request-Id': marker})
|
||||
check(error['headers']['X-Request-Id'] == marker, 'error response lost the request id')
|
||||
check(error['json']['error']['request_id'] == marker, 'error body request_id mismatch')
|
||||
|
||||
|
||||
# -------------------------------------------------------------- unknown route
|
||||
@test
|
||||
def unknown_route_and_method_are_json_404(ctx):
|
||||
ctx.client.call('GET', '/api/v1/unknown', 404)
|
||||
ctx.client.call('DELETE', '/api/v1/chats', 404, actor=ctx.alice)
|
||||
ctx.client.call('GET', '/api/v1/chats/', 404, actor=ctx.alice)
|
||||
|
||||
|
||||
# --------------------------------------------------------------- user payload
|
||||
@test
|
||||
def malformed_json_is_rejected(ctx):
|
||||
json_headers = {'Content-Type': 'application/json'}
|
||||
ctx.client.call('POST', '/api/v1/users', 400, raw=b'{"username": "broken",', headers=json_headers)
|
||||
ctx.client.call('POST', '/api/v1/users', 400, raw=b'[1, 2, 3]', headers=json_headers)
|
||||
ctx.client.call('POST', '/api/v1/users', 400, raw=b'"just a string"', headers=json_headers)
|
||||
ctx.client.call('POST', '/api/v1/users', 400, raw=b'', headers=json_headers)
|
||||
ctx.client.call('POST', '/api/v1/users', 415, raw=b'{"username": "x"}',
|
||||
headers={'Content-Type': 'text/plain'})
|
||||
ctx.client.call('POST', '/api/v1/users', 415, raw=b'{"username": "x"}',
|
||||
headers={'Content-Type': 'application/x-www-form-urlencoded'})
|
||||
|
||||
|
||||
@test
|
||||
def duplicate_username_is_409(ctx):
|
||||
name = 'dup_' + uuid.uuid4().hex[:12]
|
||||
body = {'username': name, 'display_name': 'Дубликат',
|
||||
'password': 'T3st-' + uuid.uuid4().hex}
|
||||
created = ctx.client.call('POST', '/api/v1/users', 201, body=body, schema='User')['json']
|
||||
check(created['username'] == body['username'], 'username changed on creation')
|
||||
conflict = ctx.client.call('POST', '/api/v1/users', 409, body=body)
|
||||
check(conflict['json']['error']['code'] == 'conflict', 'duplicate username must be conflict')
|
||||
|
||||
|
||||
@test
|
||||
def user_payload_validation(ctx):
|
||||
name = 'valid_' + uuid.uuid4().hex[:8]
|
||||
for bad in (
|
||||
{'username': name, 'display_name': 'Имя', 'password': 'ok', 'extra': 1},
|
||||
{'display_name': 'Имя', 'password': 'ok'},
|
||||
{'username': name, 'password': 'ok'},
|
||||
{'username': name, 'display_name': ' ', 'password': 'ok'},
|
||||
{'username': name, 'display_name': '', 'password': 'ok'},
|
||||
{'username': name, 'display_name': 42, 'password': 'ok'},
|
||||
{'username': name, 'display_name': None, 'password': 'ok'},
|
||||
{'username': name, 'display_name': 'д' * 101, 'password': 'ok'},
|
||||
{'username': 'AB', 'display_name': 'Имя', 'password': 'ok'},
|
||||
{'username': 'with space', 'display_name': 'Имя', 'password': 'ok'},
|
||||
{'username': 'юзер123', 'display_name': 'Имя', 'password': 'ok'},
|
||||
{'username': 'x' * 33, 'display_name': 'Имя', 'password': 'ok'},
|
||||
{'username': '', 'display_name': 'Имя', 'password': 'ok'},
|
||||
{'username': 123, 'display_name': 'Имя', 'password': 'ok'},
|
||||
# password too short
|
||||
{'username': name, 'display_name': 'Имя', 'password': 'short'},
|
||||
# password too long
|
||||
{'username': name, 'display_name': 'Имя', 'password': 'д' * 129},
|
||||
):
|
||||
ctx.client.call('POST', '/api/v1/users', 400, body=bad)
|
||||
ok = ctx.client.call('POST', '/api/v1/users', 201,
|
||||
body={'username': name, 'display_name': 'Студент Юникод 👋',
|
||||
'password': 'T3st-' + uuid.uuid4().hex},
|
||||
schema='User')['json']
|
||||
check(ok['display_name'] == 'Студент Юникод 👋', 'unicode display_name changed')
|
||||
check('password' not in ok and 'token' not in ok, 'user projection leaks credentials')
|
||||
|
||||
|
||||
# ------------------------------------------------------------- sessions
|
||||
@test
|
||||
def basic_login_issues_bearer_session(ctx):
|
||||
reg = ctx.alice['registration']
|
||||
credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode()
|
||||
session = ctx.client.call('POST', '/api/v1/auth/sessions', 201,
|
||||
headers={'Authorization': 'Basic ' + credential},
|
||||
schema='Session')['json']
|
||||
check(session['user'] == ctx.alice['user'], 'session user mismatch')
|
||||
check(session['token_type'] == 'Bearer', 'token type must be Bearer')
|
||||
check(len(session['token']) >= 32, 'token must be opaque and long enough')
|
||||
check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),
|
||||
'session already expired')
|
||||
ctx.alice['token'] = session['token']
|
||||
|
||||
# wrong password -> 401
|
||||
bad = base64.b64encode((reg['username'] + ':wrong-password').encode()).decode()
|
||||
ctx.client.call('POST', '/api/v1/auth/sessions', 401,
|
||||
headers={'Authorization': 'Basic ' + bad})
|
||||
# unknown user -> 401
|
||||
bad = base64.b64encode(('nonexistent-user:' + reg['password']).encode()).decode()
|
||||
ctx.client.call('POST', '/api/v1/auth/sessions', 401,
|
||||
headers={'Authorization': 'Basic ' + bad})
|
||||
|
||||
|
||||
@test
|
||||
def bearer_required_for_business_endpoints(ctx):
|
||||
# no auth -> 401
|
||||
ctx.client.call('GET', '/api/v1/users/me', 401)
|
||||
# Basic on business endpoint -> 401
|
||||
reg = ctx.alice['registration']
|
||||
credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode()
|
||||
ctx.client.call('GET', '/api/v1/users/me', 401,
|
||||
headers={'Authorization': 'Basic ' + credential})
|
||||
# X-User-Id ignored -> 401
|
||||
ctx.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': ctx.alice['id']})
|
||||
# Bearer works
|
||||
me = ctx.client.call('GET', '/api/v1/users/me', 200, actor=ctx.alice,
|
||||
schema='User')['json']
|
||||
check(me == ctx.alice['user'], 'current user mismatch')
|
||||
|
||||
|
||||
@test
|
||||
def logout_revokes_session(ctx):
|
||||
reg = ctx.alice['registration']
|
||||
credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode()
|
||||
session = ctx.client.call('POST', '/api/v1/auth/sessions', 201,
|
||||
headers={'Authorization': 'Basic ' + credential},
|
||||
schema='Session')['json']
|
||||
token = session['token']
|
||||
ctx.client.call('DELETE', '/api/v1/auth/sessions/current', 204,
|
||||
headers={'Authorization': 'Bearer ' + token})
|
||||
# revoked token -> 401
|
||||
ctx.client.call('GET', '/api/v1/users/me', 401,
|
||||
headers={'Authorization': 'Bearer ' + token})
|
||||
ctx.client.call('DELETE', '/api/v1/auth/sessions/current', 401,
|
||||
headers={'Authorization': 'Bearer ' + token})
|
||||
|
||||
|
||||
# -------------------------------------------------------------- message rules
|
||||
@test
|
||||
def foreign_chat_is_invisible(ctx):
|
||||
other = ctx.new_chat('Чат без Евы', [ctx.alice, ctx.bob])
|
||||
ctx.client.call('GET', CHAT_PATH % other['id'], 404, actor=ctx.eve)
|
||||
ctx.client.call('GET', MESSAGES_PATH % other['id'], 404, actor=ctx.eve)
|
||||
ctx.client.call('POST', MESSAGES_PATH % other['id'], 404, actor=ctx.eve,
|
||||
body={'text': 'чужое сообщение'})
|
||||
listed = ctx.client.call('GET', '/api/v1/chats?limit=100', actor=ctx.eve,
|
||||
schema='ChatPage')['json']
|
||||
check(all(item['id'] != other['id'] for item in listed['items']),
|
||||
'foreign chat leaked into the list of another user')
|
||||
member_view = ctx.client.call('GET', '/api/v1/chats?limit=100', actor=ctx.bob,
|
||||
schema='ChatPage')['json']
|
||||
check(any(item['id'] == other['id'] for item in member_view['items']),
|
||||
'member cannot see the own chat')
|
||||
ctx.client.call('GET', CHAT_PATH % uuid.uuid4(), 404, actor=ctx.alice)
|
||||
ctx.client.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=ctx.alice)
|
||||
ctx.client.call('GET', MESSAGES_PATH % 'not-a-uuid', 400, actor=ctx.alice)
|
||||
|
||||
|
||||
@test
|
||||
def message_payload_validation(ctx):
|
||||
"""Empty and whitespace-only text is rejected."""
|
||||
for bad in (
|
||||
{'text': ''},
|
||||
{'text': ' '},
|
||||
{'text': '\u00a0\t\n'},
|
||||
{},
|
||||
{'text': 123},
|
||||
{'text': None},
|
||||
{'text': []},
|
||||
{'text': 'ок', 'sender_id': ctx.bob['id']},
|
||||
{'text': 'д' * 4001},
|
||||
):
|
||||
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 400, actor=ctx.alice, body=bad)
|
||||
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 401, body={'text': 'ок'})
|
||||
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 400, actor=ctx.alice, raw=b'{"text":}',
|
||||
headers={'Content-Type': 'application/json'})
|
||||
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 415, actor=ctx.alice,
|
||||
raw=b'{"text":"ok"}', headers={'Content-Type': 'application/xml'})
|
||||
|
||||
|
||||
@test
|
||||
def text_limits_and_unicode_roundtrip(ctx):
|
||||
longest = 'д' * 3999 + '👋'
|
||||
check(len(longest) == 4000, 'fixture must be exactly 4000 code points')
|
||||
created = ctx.send(ctx.bob, ctx.chat, longest)
|
||||
check(created['text'] == longest, '4000 code points text changed')
|
||||
check(created['sender_id'] == ctx.bob['id'], 'sender must be derived from actor')
|
||||
mixed = 'Привет 👋🏽 \u00a0 e\u0301 — астральные 🚀🧪 и кавычки «»'
|
||||
echo = ctx.send(ctx.alice, ctx.chat, mixed)
|
||||
check(echo['text'] == mixed, 'mixed unicode text changed in transit')
|
||||
check(echo['chat_id'] == ctx.chat['id'], 'message bound to the wrong chat')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- pagination
|
||||
@test
|
||||
def multi_page_traversal_is_exact_and_stable(ctx):
|
||||
chat = ctx.new_chat('Пагинация', [ctx.bob, ctx.alice])
|
||||
sent = []
|
||||
for index in range(7):
|
||||
actor = ctx.alice if index % 2 == 0 else ctx.bob
|
||||
sent.append(ctx.send(actor, chat, 'страница-%d-👋' % index))
|
||||
expected = traversal_order(ordered(sent))
|
||||
walked, pages = ctx.traverse(ctx.bob, chat, [2])
|
||||
check(pages >= 4, 'seven messages with limit=2 must take at least four pages')
|
||||
check(traversal_order(walked) == expected, 'traversal lost, duplicated or reordered messages')
|
||||
check(len(set(traversal_order(walked))) == len(walked), 'traversal returned duplicates')
|
||||
switched, _ = ctx.traverse(ctx.alice, chat, [2, 3, 1])
|
||||
check(traversal_order(switched) == expected, 'changing limit mid-traversal changed the order')
|
||||
singles, _ = ctx.traverse(ctx.alice, chat, [1])
|
||||
check(len(singles) == 7, 'limit=1 traversal must return all seven messages')
|
||||
first = ctx.page(ctx.alice, chat, 2)
|
||||
check(len(first['items']) == 2 and first['next_cursor'], 'first page shape is wrong')
|
||||
check(first['items'][0]['id'] == expected[0], 'first page must start with the oldest message')
|
||||
|
||||
|
||||
@test
|
||||
def default_page_limit_is_fifty(ctx):
|
||||
chat = ctx.new_chat('Предел страницы по умолчанию', [ctx.alice, ctx.bob])
|
||||
for index in range(51):
|
||||
ctx.send(ctx.alice, chat, 'default-limit-%d' % index)
|
||||
page = ctx.client.call('GET', MESSAGES_PATH % chat['id'], actor=ctx.alice,
|
||||
schema='MessagePage')['json']
|
||||
check(len(page['items']) == 50, 'default limit must be 50, got %d' % len(page['items']))
|
||||
check(page['next_cursor'], 'a page that is not the last one needs a cursor')
|
||||
tail = ctx.page(ctx.alice, chat, 100, page['next_cursor'])
|
||||
check(len(tail['items']) == 1, 'the remaining page must hold exactly one message')
|
||||
check(tail['next_cursor'] is None, 'last page must have next_cursor null')
|
||||
merged = traversal_order(page['items']) + traversal_order(tail['items'])
|
||||
check(len(set(merged)) == 51, 'page boundaries duplicated or dropped messages')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Student-owned lab2 tests: sessions, validation, pagination')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
ctx = Context(client)
|
||||
print('student-tests: %d cases against %s' % (len(TESTS), client.base))
|
||||
failures = []
|
||||
for func in TESTS:
|
||||
try:
|
||||
func(ctx)
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
failures.append(func.__name__)
|
||||
print('FAIL %-42s %s' % (func.__name__, exc))
|
||||
else:
|
||||
print('PASS %-42s' % func.__name__)
|
||||
print('%d passed, %d failed' % (len(TESTS) - len(failures), len(failures)))
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user