Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf4a7ad142 |
@@ -9,3 +9,11 @@ DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# --- реализация (см. REPORT.md, «Соответствие переменных») ---
|
||||
# Порт API на хосте (публикуется только на 127.0.0.1) и порт экземпляра с коротким TTL (профиль ttl).
|
||||
API_PORT=8080
|
||||
API_TTL_PORT=8081
|
||||
# Redis используется как read-through кэш сессий; источник истины — PostgreSQL.
|
||||
SESSION_CACHE=redis
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
DB_POOL_MAX=10
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# Shell-скрипты и файлы, читаемые внутри Linux-контейнеров, — всегда LF (иначе ломаются на Windows с autocrlf).
|
||||
*.sh text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.py text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
.env.example text eol=lf
|
||||
# Windows-обёртки
|
||||
*.cmd text eol=crlf
|
||||
*.ps1 text eol=crlf
|
||||
@@ -0,0 +1,31 @@
|
||||
name: Application stack
|
||||
# Отдельный job решения: сборка образа API, запуск Compose-стенда, публичный smoke,
|
||||
# тест сохранности с restart hook и собственные проверки студента. Секреты генерируются
|
||||
# на лету (синтетические), в репозитории их нет.
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
stack:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Secrets and env (synthetic)
|
||||
run: ./scripts/init-secrets.sh
|
||||
- name: Compose config
|
||||
run: docker compose --env-file .env config --quiet
|
||||
- name: Build and start (api, api-ttl, postgres, redis)
|
||||
run: docker compose --env-file .env --profile ttl up -d --build --wait
|
||||
- name: Public smoke
|
||||
run: make test
|
||||
- name: Persistence (restart API + PostgreSQL + Redis, volumes kept)
|
||||
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh && cat evidence/restart-hook.txt
|
||||
- name: Student tests (API/auth/validation/race/rollback, expiry, dependency failures)
|
||||
run: make test-student
|
||||
- name: Logs on failure
|
||||
if: failure()
|
||||
run: docker compose --env-file .env --profile ttl logs --no-color --tail=200
|
||||
- name: Stop (volumes are discarded with the runner)
|
||||
if: always()
|
||||
run: docker compose --env-file .env --profile ttl --profile research down -v
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
# API мессенджера lab2. Python 3.13, зависимости ставятся в venv в отдельной стадии.
|
||||
ARG PYTHON_IMAGE=python:3.13.5-slim-bookworm
|
||||
|
||||
FROM ${PYTHON_IMAGE} AS deps
|
||||
ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
WORKDIR /build
|
||||
COPY requirements.txt ./
|
||||
RUN python -m venv /opt/venv \
|
||||
&& /opt/venv/bin/pip install --require-virtualenv --no-compile -r requirements.txt
|
||||
|
||||
FROM ${PYTHON_IMAGE}
|
||||
ENV PATH=/opt/venv/bin:$PATH \
|
||||
PYTHONPATH=/app/src \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
|
||||
COPY --from=deps /opt/venv /opt/venv
|
||||
WORKDIR /app
|
||||
COPY migrations ./migrations
|
||||
COPY src ./src
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=30s --retries=6 \
|
||||
CMD ["python", "-m", "messenger", "healthcheck"]
|
||||
# PID 1 — сам HTTP-процесс (exec-форма), SIGTERM даёт graceful shutdown uvicorn.
|
||||
ENTRYPOINT ["python", "-m", "messenger"]
|
||||
CMD ["serve"]
|
||||
@@ -24,3 +24,61 @@ load:
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Цели решения (lab2). Публичные цели выше не изменены.
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
COMPOSE ?= docker compose --env-file .env
|
||||
TTL_BASE_URL ?= http://localhost:8081
|
||||
|
||||
.PHONY: secrets up up-ttl down ps logs destroy-data test-student test-api test-expiry test-failures test-cache-race \
|
||||
research-track2 research-up evidence
|
||||
|
||||
secrets:
|
||||
./scripts/init-secrets.sh
|
||||
|
||||
up: secrets
|
||||
$(COMPOSE) up -d --build --wait
|
||||
|
||||
up-ttl:
|
||||
$(COMPOSE) --profile ttl up -d --build --wait api-ttl
|
||||
|
||||
ps:
|
||||
$(COMPOSE) --profile ttl --profile research ps
|
||||
|
||||
logs:
|
||||
$(COMPOSE) logs --tail=100 api
|
||||
|
||||
# Безопасная остановка: контейнеры удаляются, именованные volumes (данные) сохраняются.
|
||||
down:
|
||||
$(COMPOSE) --profile ttl --profile research down
|
||||
|
||||
# НАМЕРЕННОЕ удаление учебных данных (volumes pgdata/pgrestore). Необратимо.
|
||||
destroy-data:
|
||||
@echo "This deletes ALL lab2 data volumes. Type 'yes' to continue:"; read ans; [ "$$ans" = yes ]
|
||||
$(COMPOSE) --profile ttl --profile research down -v
|
||||
|
||||
test-api:
|
||||
$(PYTHON) student-tests/test_api.py
|
||||
|
||||
test-expiry:
|
||||
TTL_BASE_URL="$(TTL_BASE_URL)" $(PYTHON) student-tests/test_expiry.py
|
||||
|
||||
# Останавливает/запускает контейнеры redis и postgres (volumes не трогает).
|
||||
test-failures:
|
||||
$(PYTHON) student-tests/test_failures.py
|
||||
|
||||
# In-process проверка гонки кэш/logout внутри контейнера API (сеть backend, реальный Redis).
|
||||
test-cache-race:
|
||||
$(COMPOSE) run --rm --no-deps -T -v "$(CURDIR)/student-tests:/app/student-tests:ro" --entrypoint python api student-tests/test_cache_race.py
|
||||
|
||||
test-student: test-api test-expiry test-cache-race test-failures
|
||||
|
||||
research-up:
|
||||
$(COMPOSE) --profile research up -d --wait postgres-restore
|
||||
|
||||
research-track2: research-up
|
||||
$(PYTHON) research/track2.py
|
||||
|
||||
evidence:
|
||||
./scripts/collect-evidence.sh
|
||||
|
||||
@@ -1,37 +1,225 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: **TODO: ФИО, группа**
|
||||
- Целевая оценка; выбранный трек 5: **5, трек 2 «План запроса и восстановление»**
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники, библиотеки и помощь ИИ:
|
||||
- библиотеки: Starlette 1.7.0, uvicorn 0.54.0, psycopg 3.2.10 + psycopg_pool 3.2.6, redis-py 5.2.1; KDF — `hashlib.scrypt` (OpenSSL) из стандартной библиотеки Python;
|
||||
- документация: PostgreSQL (transactions, EXPLAIN, row comparison, `pg_dump`/`pg_restore`), Redis persistence и Lua scripting, RFC 7617, OWASP Password Storage Cheat Sheet (параметры scrypt);
|
||||
- **ИИ:** реализация, тесты, скрипты исследования и этот отчёт подготовлены с помощью ИИ-ассистента Claude (Anthropic). Чужой код из других решений не использовался.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Prerequisites:** Docker Engine 24+ с Compose v2, Python 3.9+ на хосте (для публичных и собственных тестов, только stdlib), `make` (Linux/macOS; для Windows — PowerShell-скрипты ниже). Свободны порты `127.0.0.1:8080` (API) и `127.0.0.1:8081` (экземпляр с коротким TTL).
|
||||
|
||||
**Образы (закреплены тегами):** `python:3.13.5-slim-bookworm` (база API), `postgres:16.10-alpine`, `redis:7.4.2-alpine`. API собирается из исходников этого репозитория (`Dockerfile`), зависимости закреплены в `requirements.txt`.
|
||||
|
||||
```sh
|
||||
# 1. Локальные синтетические secrets и .env (значения не печатаются, существующие не перезаписываются)
|
||||
./scripts/init-secrets.sh # = make secrets; создаёт secrets/* и .env из .env.example
|
||||
|
||||
# 2. Сборка, миграции (применяются API при старте под advisory lock), ожидание healthcheck
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait # = make up
|
||||
curl -s localhost:8080/health/ready # {"status":"ok"}
|
||||
|
||||
# 3. Проверки
|
||||
make check
|
||||
make test # публичный smoke
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # рестарт API+PostgreSQL+Redis с volumes
|
||||
make up-ttl # api-ttl с SESSION_TTL_SECONDS=2 на :8081
|
||||
make test-student # test-api, test-expiry, test-cache-race, test-failures (останавливает redis/postgres!)
|
||||
make research-track2 # трек 5.2: отдельная БД research + сервер postgres-restore (профиль research)
|
||||
./scripts/collect-evidence.sh # всё сразу, вывод в evidence/
|
||||
|
||||
# 4. Остановка
|
||||
make down # безопасно: контейнеры удаляются, volumes pgdata/pgrestore сохраняются
|
||||
make destroy-data # НАМЕРЕННОЕ удаление всех учебных данных (down -v), спрашивает подтверждение
|
||||
```
|
||||
|
||||
**Windows (PowerShell, без `make` и WSL).** Нужны Docker Desktop (запущен) и Python 3.9+ с python.org. Из корня репозитория:
|
||||
|
||||
```powershell
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\init-secrets.ps1 # secrets\ и .env
|
||||
docker compose --env-file .env --profile ttl up -d --build --wait
|
||||
py -3 tests\smoke.py --base-url http://localhost:8080 # публичный smoke
|
||||
py -3 tests\resilience.py --mode persistence --action-script scripts\restart.cmd
|
||||
# Всё сразу (up + все проверки + трек 5.2), вывод в evidence\:
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\collect-evidence.ps1
|
||||
docker compose --env-file .env --profile ttl --profile research down # остановка, volumes сохраняются
|
||||
```
|
||||
|
||||
`scripts\restart.cmd` — обёртка над `scripts\restart.ps1` (тот же хук, что `restart.sh`): `tests/resilience.py` запускает хук напрямую, а `.cmd` Windows исполняет без shell. В Windows PowerShell 5.1 нет оператора `&&` — команды выполняются по одной строке. Ключи `collect-evidence.ps1`: `-SkipFailures` (не останавливать redis/postgres), `-SkipResearch`.
|
||||
|
||||
Действия оператора: только шаг 1 (создать secrets). Миграции вручную запускать не нужно; повторно — `docker compose --env-file .env exec api python -m messenger migrate` (no-op, если всё применено). Пароль БД зафиксирован в volume при первой инициализации: не удаляйте `secrets/` при живом volume `pgdata`.
|
||||
|
||||
> **Важно о доказательствах в `evidence/`.** Решение разрабатывалось в среде, где Docker daemon работал, но Docker Hub/зеркала и PyPI были закрыты политикой сети. Поэтому `evidence/*` собраны на **нативном стенде** с теми же компонентами: PostgreSQL 16.15, Redis 7.0.15, тот же код API под Python 3.13.17 (`scripts/dev/*` — хуки и сборщик для такого стенда; `evidence/stack.txt` — версии и процессы). Compose-файл проверен `docker compose config`, раскладка контейнера (код только для чтения, uid 10001, `/app/src` + `/app/migrations`) проверена запуском API в такой же раскладке. Перед защитой выполните `make up && ./scripts/collect-evidence.sh` (Windows: `scripts\collect-evidence.ps1`, см. выше), чтобы `evidence/` содержал вывод именно Compose-стенда; тот же сценарий автоматически гоняет `.github/workflows/app.yml`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
127.0.0.1:8080 сеть backend (internal: true)
|
||||
клиент ──HTTP──► api (Starlette/uvicorn, 1 процесс) ──► postgres:16 [volume pgdata] источник истины
|
||||
│ пул psycopg ≤10 соединений users, chats, chat_members, messages,
|
||||
│ sessions, session_cache_invalidations
|
||||
└──────────────────────────────────► redis:7.4 [без volume] кэш сессий (TTL)
|
||||
127.0.0.1:8081 ──► api-ttl (профиль ttl, SESSION_TTL_SECONDS=2) — только для теста истечения
|
||||
postgres-restore (профиль research) [volume pgrestore] — «новое хранилище» трека 5.2
|
||||
```
|
||||
|
||||
| Роль | Service | Образ | Health/readiness | Volume | Сеть/порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| api | `api` | сборка `Dockerfile` → `mtusi-lab2-api:local` | `python -m messenger healthcheck` → `/health/ready` | нет (stateless) | frontend+backend, `127.0.0.1:8080` | secrets `database_url`, `cache_url` → `*_FILE` |
|
||||
| postgres | `postgres` | `postgres:16.10-alpine` | `pg_isready` | `pgdata` | backend, не публикуется | `POSTGRES_PASSWORD_FILE` |
|
||||
| redis_or_valkey | `redis` | `redis:7.4.2-alpine` | `redis-cli ping` (пароль через `REDISCLI_AUTH`) | нет — это кэш | backend | `requirepass` из secret, конфиг пишется в `/tmp` |
|
||||
|
||||
Контейнер API: non-root (uid 10001), `read_only`, `cap_drop: ALL`, `no-new-privileges`, лимиты 1 CPU / 512 MiB; PID 1 — сам uvicorn, SIGTERM → graceful shutdown.
|
||||
|
||||
**Что перенесено из lab1 и что изменено.** Предметная модель и REST-контракт сохранены; хранилище in-memory заменено PostgreSQL. По объявленному переходу 1→2: `password` обязателен при регистрации, `POST /auth/sessions` принимает только Basic, все бизнес-endpoint — только Bearer; `X-User-Id` игнорируется (не даёт доступа и не меняет владельца).
|
||||
|
||||
**Модель данных** (`migrations/0001_init.sql`): PK у всех таблиц; `users.username` — `UNIQUE` + `CHECK` формата; `chat_members (chat_id, user_id)` — PK и FK на `chats`/`users`, `position` сохраняет порядок `member_ids`; `messages` имеет составной FK `(chat_id, sender_id) → chat_members`, т.е. «отправитель — участник чата» гарантирует сама БД; CHECK длин (`char_length` — code points). Время назначает БД (`clock_timestamp()`), отдаётся как RFC 3339 UTC с микросекундами и `Z`. Индексы выводятся из запросов: `messages (chat_id, created_at, id)` — история; `chat_members (user_id, chat_id)` — список чатов пользователя; `sessions (expires_at)` — очистка.
|
||||
|
||||
**Миграции:** `src/messenger/db.py` применяет `migrations/NNNN_*.sql` по порядку, каждую в своей транзакции, и записывает версию+SHA-256 в `schema_migrations`; всё под `pg_advisory_lock` (несколько экземпляров API не применят миграцию дважды — `evidence/migrations.txt`: два экземпляра стартовали одновременно на пустой БД, один применил обе миграции, второй — `applied: []`). Изменённая уже применённая миграция — ошибка старта. Seed-данных нет, поэтому повторный запуск ничего не дублирует.
|
||||
|
||||
**Транзакционные границы.**
|
||||
- `POST /chats`: одна транзакция `INSERT chats` + `INSERT chat_members … unnest(member_ids)`. Неизвестный участник → FK violation → ROLLBACK → 400; чат без участников не возникает (проверено тестом).
|
||||
- `POST /messages`: один оператор `INSERT … SELECT … WHERE EXISTS(membership)` — проверка доступа и запись атомарны; плюс составной FK.
|
||||
- Регистрация: уникальный индекс — окончательный арбитр гонки (предварительный `SELECT` лишь экономит KDF).
|
||||
- `DELETE /auth/sessions/current`: одна транзакция `UPDATE sessions SET revoked_at` + строка в outbox инвалидации кэша. COMMIT в PostgreSQL делает отзыв окончательным.
|
||||
|
||||
**Аутентификация и сессии.** Пароль: scrypt `N=2^15, r=8, p=3` (вариант OWASP, ≈32 MiB на вычисление), соль 16 байт, формат `$scrypt$ln=15,r=8,p=3$salt$hash` — параметры можно усилить без потери старых hash. Одновременных KDF ≤ `KDF_CONCURRENCY=4` (защита RAM). Для неизвестного username выполняется KDF над фиктивным hash — время ответа не выдаёт существование пользователя; ответ 401 одинаковый. Токен — `secrets.token_urlsafe(32)` (256 бит), в БД хранится только `SHA-256(token)` (быстрый hash уместен: токен не перебирается, в отличие от пароля). TTL — `SESSION_TTL_SECONDS`, `expires_at` вычисляется в БД и после рестарта не продлевается.
|
||||
|
||||
**Роль Redis (уровень 4): read-through кэш сессий; источник истины — PostgreSQL.**
|
||||
- Ключ `sess:v1:<sha256>` → `{user, exp}`; TTL записи = `min(SESSION_CACHE_TTL_SECONDS=60, остаток сессии)` — кэш не продлевает сессию (проверено, `pttl=1198ms` при остатке 1.2 с).
|
||||
- Инвалидация при logout: после COMMIT — Lua-скрипт ставит «надгробие» `sess:v1:revoked:<hash>` (TTL 75 с) и удаляет ключ. Заполнение кэша — Lua `SET`, только если надгробия нет. Это закрывает гонку «запрос прочитал сессию из PG до logout и пишет её в кэш после logout» (`evidence/test-cache-race.txt`).
|
||||
- Если Redis недоступен при logout, строка outbox `session_cache_invalidations` остаётся; любая ошибка Redis переводит экземпляр в режим `degraded` — кэш не читается, аутентификация идёт в PostgreSQL. Фоновый цикл (раз в 1 с) после успешного `PING` дообрабатывает outbox и только затем снова включает чтение кэша (тест: outbox=1 во время отказа → 0 после восстановления, отозванный токен остаётся 401).
|
||||
- Persistence у Redis выключена намеренно (`--save '' --appendonly no`): flush/restart Redis = холодный кэш, все сессии в PostgreSQL. Redis не критичен для `/health/ready`.
|
||||
|
||||
**Пул и таймауты.** `AsyncConnectionPool(min=1, max=10, timeout=2s, max_waiting=200, check=check_connection)`; `connect_timeout=3s`, `statement_timeout=5s`, `lock_timeout=2s`, `idle_in_transaction_session_timeout=10s`; Redis `socket_timeout=0.2s`; общий таймаут запроса 10 с; лимит тела 64 KiB (413). Все SQL — параметризованные (psycopg передаёт параметры отдельно от текста, server-side binding); динамически подставляется только выбор из двух литеральных шаблонов SQL.
|
||||
|
||||
**Ошибки.** Единый формат с `request_id` = `X-Request-Id` в каждом ответе (включая 404/405 роутера). Недоступность БД/истечение пула/таймаут → 503 `unavailable` + `Retry-After: 1`; прочее → 500 без деталей. 405 отдаётся с кодом `invalid_request` (в enum контракта нет отдельного кода). Логи — JSON в stdout, без query-строк, заголовков, тел, токенов и паролей.
|
||||
|
||||
**Пагинация.** Keyset по `(created_at, id)`, `limit+1` для определения `next_cursor`. Cursor — base64url(JSON: вид списка, область — chat_id или user_id, позиция); cursor другого чата/пользователя → 400. Конкурентные вставки: `created_at` назначается `clock_timestamp()` в момент INSERT; транзакция с меньшим временем может закоммититься позже, и читатель, уже прошедший эту позицию, её не увидит при продолжении обхода (повторов нет, пропуск возможен только для записей, закоммиченных «в прошлое» во время обхода). Snapshot между HTTP-запросами контракт не требует.
|
||||
|
||||
**Соответствие переменных `.env.example` → реализация.**
|
||||
|
||||
| Переменная | Где используется | Значение в Compose |
|
||||
| --- | --- | --- |
|
||||
| `APP_HOST`, `APP_PORT` | адрес uvicorn | `0.0.0.0`, `8080` |
|
||||
| `APP_ENV` | метка окружения | `lab` |
|
||||
| `SESSION_TTL_SECONDS` | срок сессии | `3600` (api), `2` (api-ttl) |
|
||||
| `DATABASE_URL_FILE` | DSN PostgreSQL из secret | `/run/secrets/database_url` |
|
||||
| `SESSION_STORE` | только `postgres` (источник истины) | `postgres` |
|
||||
| `CACHE_URL_FILE` | URL Redis из secret | `/run/secrets/cache_url` |
|
||||
| `SESSION_CACHE` (доп.) | `redis` / `off` / `auto` | `redis` |
|
||||
| `SESSION_CACHE_TTL_SECONDS`, `DB_POOL_*`, `DB_STATEMENT_TIMEOUT_MS`, `CACHE_TIMEOUT_SECONDS`, `REQUEST_TIMEOUT_SECONDS`, `KDF_CONCURRENCY`, `SCRYPT_*` (доп.) | см. `src/messenger/config.py` | значения по умолчанию |
|
||||
| `API_PORT`, `API_TTL_PORT` (доп.) | порты на хосте | `8080`, `8081` |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Все файлы — в `evidence/`. Стенд, на котором они получены, — см. примечание выше и `evidence/stack.txt`.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `make test` | `PASS: lab2 public API scenario` | `smoke.txt` |
|
||||
| Структура skeleton | `make check` | PASS | `make-check.txt` |
|
||||
| 3: данные и действующие сессии переживают рестарт API и хранилищ без удаления volumes | `make test-persistence ACTION_SCRIPT=…` | PASS тем же токеном; PID API/PostgreSQL/Redis до и после различаются | `persistence.txt`, `restart-hook.txt` |
|
||||
| 3: PK/FK, UNIQUE username, миграции с нуля, повтор без дублей | старт API на пустой БД; `python -m messenger migrate` | 2 миграции применены, повтор `{"applied": []}`, 13 индексов | `migrations.txt` |
|
||||
| 3: атомарное создание чата | `test-api`: «compound create-chat rolls back» | 400, 0 строк чата с этим title, 0 «сиротских» участников | `test-api.txt` |
|
||||
| 3: Basic только на `/auth/sessions`, Bearer везде, X-User-Id не даёт доступа, expiry и logout | `test-api`, smoke | 401 + верный `WWW-Authenticate`; X-User-Id не меняет пользователя; logout → 401, повтор → 401 | `test-api.txt`, `smoke.txt` |
|
||||
| 3: password KDF | `test-api`: «stored only as scrypt» | `$scrypt$ln=15,r=8,p=3$…`, у одинаковых паролей разные hash | `test-api.txt` |
|
||||
| 3: secrets/volumes вне Git | `.gitignore` (`secrets/`, `.env`, `backups/`) | в архиве нет `secrets/`, `.env`, дампов | — |
|
||||
| 4: Redis с конкретной ролью, TTL, инвалидация | `test-cache-race`, `test-failures` | надгробие блокирует поздний SET; TTL ≤ остатка сессии; outbox дообработан после восстановления | `test-cache-race.txt`, `test-failures.txt` |
|
||||
| 4: параметризованные запросы, ограниченный пул, таймауты | `test-api`: injection-payload; конфигурация пула | payload сохранён дословно; при остановленной БД ответы 503 за ≈2 с (таймаут пула) | `test-api.txt`, `test-failures.txt` |
|
||||
| 4: гонка регистрации | `test-api`: 20 параллельных POST одного username | ровно 1×201 и 19×409, 1 строка; все 19 конфликтов отсечены уникальным индексом | `test-api.txt`, `race-unique-violations.txt` (=19) |
|
||||
| 4: rollback составной операции | см. строку уровня 3 | — | `test-api.txt` |
|
||||
| 4: индекс истории + EXPLAIN (ANALYZE, BUFFERS) на ≥10 000 сообщений; пагинация без полного scan | `research/track2.py plans` на 120 000 сообщений | keyset-страница из середины горячего чата: Index Scan, 12 буферов, 0.033 мс | `track2/explain_V3_chat_created_id_hot.txt`, `track2/plans.json` |
|
||||
| 4: автоматические проверки expiry, logout, неверного пароля, недоступности Redis/БД | `test-expiry`, `test-api`, `test-failures` | все PASS; аноним при любом отказе — 401, неизвестный токен при мёртвой БД — не 200 | `test-expiry.txt`, `test-api.txt`, `test-failures.txt` |
|
||||
| 5, трек 2: ≥100 000 сообщений, неравномерные чаты, ≥2 индекса/запроса, стоимость записи | `research/track2.py` | 5 вариантов индекса × 3 запроса × 3 чата; WAL/insert и размер индекса | `track2/dataset.json`, `plans.json`, `writes.json` |
|
||||
| 5, трек 2: backup → новое хранилище → сверка | `research/track2.py backup` | все таблицы, индексы, ограничения и 200 выбранных сообщений совпали; граница потери данных измерена | `track2/backup_restore.json` |
|
||||
| Нагрузка (для профиля) | `tests/load.py --duration 30 --concurrency 4` | 598 успешных RPS, 0 ошибок, p50/p95/p99 = 6.3/9.4/12.0 мс | `load.json`, `load-resources.json` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Окружение измерений:** 2 vCPU Intel Xeon 2.8 GHz, 8 GiB RAM, без cgroup-лимитов; PostgreSQL 16.15 c `shared_buffers=128MB` (по умолчанию), Redis 7.0.15; один процесс API. Нагрузка: closed-loop `tests/load.py`, 4 потока, 30 с, без прогрева (первые запросы включены), история одного чата растёт до ~18 000 сообщений за прогон. Результат: 597.7 RPS успешных, 0 ошибок, p50 6.28 / p95 9.45 / p99 12.03 мс; API занимает ≈1 ядро (0.99) и до 80 MiB RSS — узкое место — однопроцессный Python, а не БД. Генератор closed-loop страдает coordinated omission; число не является пропускной способностью системы.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Рестарт API + PostgreSQL + Redis (restart hook) | данные и прежний токен живы | PASS, тот же токен читает те же сообщения; новые PID у всех трёх | ≈3 с до `/health/ready`; потерь нет |
|
||||
| Остановка Redis | вход, чтение, запись и logout работают через PostgreSQL | всё 2xx; задержка `/users/me` 10 мс; `/health/ready` 200 | после старта Redis outbox (1 строка) дообработан за ≤1 с; отозванный токен остаётся 401 |
|
||||
| Остановка PostgreSQL | ограниченная ошибка, без доверия анониму | `/health/live` 200, `/health/ready` 503 за 1.5 с; запись/история/вход/регистрация — 503 `unavailable` за 2.0 с; без токена — 401 | `ready` снова 200 через 2.2 с после старта БД; данные не изменились |
|
||||
| Короткий TTL = 2 с | 401 после истечения даже при прогретом кэше | PASS; строка сессии не удалена и не отозвана — истекла по `expires_at` | — |
|
||||
|
||||
### Трек 5.2. План запроса истории и восстановление
|
||||
|
||||
**Гипотезы.**
|
||||
H1. На неравномерных чатах индекс только по `chat_id` достаточен для маленьких чатов, но для горячего чата требует чтения и сортировки всей его истории на каждую страницу; составной `(chat_id, created_at, id)` делает стоимость keyset-страницы независимой от размера чата и глубины.
|
||||
H2. OFFSET-пагинация на глубине k читает O(k) записей при любом B-tree без покрытия.
|
||||
H3. Каждый дополнительный столбец индекса увеличивает стоимость записи (WAL, размер), покрывающий индекс — заметно.
|
||||
H4. Логический backup (`pg_dump`) восстанавливается в новое хранилище с полным совпадением данных, а потеря при восстановлении = все транзакции, закоммиченные после снимка `pg_dump`.
|
||||
|
||||
**Методика (повторяемая).** `python3 research/track2.py` (по умолчанию все шаги; Compose-команды подставляются сами, для нативного стенда — переменные `SRC_PSQL`/`DST_PSQL`/… см. `scripts/dev/collect-native.sh`).
|
||||
1. `setup`: отдельная БД `research`, те же миграции, что у приложения (рабочая БД не трогается).
|
||||
2. `generate` (`research/generate.sql`, `setseed(0.42)`): 2 000 пользователей, 300 чатов по 2–50 участников, **120 000 сообщений**; доля чата ранга k ∝ 1/k^1.1. Получено: крупнейший чат 24 294 (20.2 %), топ-10 — 54.4 %, медиана 98, минимум 38. Сообщения вставлены в глобальном хронологическом порядке, чаты перемешаны во времени — строки горячего чата лежат на 3 614 из ~3 620 страниц heap.
|
||||
3. `plans`: для каждого варианта индекса — `DROP/CREATE INDEX`, `VACUUM ANALYZE`, затем `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` трёх запросов для горячего, медианного и самого маленького чата: первая keyset-страница, keyset-страница с cursor в середине истории и та же страница через `OFFSET n/2`. Один прогрев + 15 измерений, медиана Execution Time. Буферы — из последнего прогона (кэш тёплый, `shared read = 0`).
|
||||
4. `writes`: 50 000 однострочных INSERT в таблицу той же формы с тем же индексом (одна транзакция, `CHECKPOINT` перед прогоном, распределение chat_id как в истории), 5 прогонов, медиана; WAL — по `pg_current_wal_insert_lsn()`.
|
||||
5. `backup`: фоновый писатель коммитит пронумерованные `marker-NNNNNN` каждые ~10 мс; через 1 с запускается `pg_dump -Fc -Z 6`, писатель продолжает ещё 1 с после окончания дампа. Затем `pg_restore -j 4` в **другой сервер** PostgreSQL (Compose: `postgres-restore` со своим volume), сверка: количество и md5-агрегат каждой таблицы, определения индексов и ограничений, 200 выбранных сообщений (первое/последнее в 10 крупнейших чатах + 180 псевдослучайных по `md5(id)`), страница истории горячего чата с OFFSET 1000, и множество маркеров.
|
||||
|
||||
**Результаты: чтение** (медиана, мс / прочитано буферов по 8 KiB; `evidence/track2/plans.json`).
|
||||
|
||||
| Вариант индекса истории | Размер | hot: keyset 1-я стр. | hot: keyset середина | hot: OFFSET 12 147 | median: keyset сер. | small: keyset сер. |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| V0 нет (только PK) | 0 | 18.13 / 3705 | 18.34 / 3647 | 28.46 / 3617 | 15.64 / 3647 | 17.62 / 3647 |
|
||||
| V1 `(chat_id)` | 0.8 MiB | 12.67 / 3636 | 12.08 / 3636 | 20.12 / 3636 | 0.081 / 99 | 0.038 / 40 |
|
||||
| V2 `(chat_id, created_at)` | 4.7 MiB | 0.036 / 10 | 0.038 / 11 | 21.45 / 3737 | 0.070 / 53 | 0.048 / 23 |
|
||||
| **V3 `(chat_id, created_at, id)`** | 6.8 MiB | **0.023 / 10** | **0.033 / 12** | 25.01 / 3790 | 0.050 / 52 | 0.028 / 22 |
|
||||
| V4 V3 `INCLUDE (sender_id, text)` | 30.0 MiB | 0.031 / 5 | 0.035 / 6 | 2.14 / 390 | 0.034 / 5 | 0.022 / 4 |
|
||||
|
||||
Планы (горячий чат): V0 — Parallel Seq Scan + Sort; V1 — Bitmap Index Scan + Bitmap Heap Scan + Sort всех 24 294 строк; V2 — Index Scan + **Incremental Sort** (в индексе нет `id`, сортировка по tie-breaker внутри одинаковых `created_at`); V3 — чистый `Index Scan` с `Index Cond: chat_id = … AND ROW(created_at, id) > ROW(…)` и `Limit` (полный текст — `explain_V3_chat_created_id_hot.txt`); V4 — Index Only Scan. Для OFFSET при V1–V3 планировщик выбирает bitmap-скан всех строк чата + sort (и даже `external merge` на диск 5.4 MB), потому что строки чата разбросаны по всей heap и случайный доступ к 12 000 строкам дороже.
|
||||
|
||||
**Результаты: запись** (`evidence/track2/writes.json`).
|
||||
|
||||
| Вариант | мкс/INSERT (медиана) | к V0 | WAL, байт/INSERT | индекс после 50 000 строк |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| V0 | 12.81 | — | 360 | — |
|
||||
| V1 | 15.56 | +21 % | 438 (+22 %) | 0.44 MiB |
|
||||
| V2 | 15.44 | +21 % | 471 (+31 %) | 3.13 MiB |
|
||||
| V3 | 14.88 | +16 % | 502 (+39 %) | 4.52 MiB |
|
||||
| V4 | 18.58 | +45 % | 849 (+136 %) | 20.95 MiB |
|
||||
|
||||
Разброс времени между прогонами V1–V3 (660–996 мс на 50 000 строк) больше разницы между ними, поэтому по CPU-времени V1, V2, V3 неразличимы; устойчивая метрика — WAL и размер: каждый столбец ключа добавляет ~30–35 байт WAL на вставку, покрывающий индекс с `text` — более чем удваивает WAL и в ≈4.6 раза больше по размеру.
|
||||
|
||||
**Результаты: backup → restore → сверка** (`evidence/track2/backup_restore.json`).
|
||||
|
||||
| Этап | Значение |
|
||||
| --- | --- |
|
||||
| `pg_dump -Fc -Z 6` (120 000 сообщений, 28.3 MiB heap) | 0.78 с, 7.2 MB |
|
||||
| `pg_restore -j 4` в новый сервер (включая построение 13 индексов и 23 ограничений) | 0.97 с |
|
||||
| Сверка | 2.37 с; совпали users 2000, chats 300, chat_members 7720, messages 120 000, schema_migrations 2, индексы 13, ограничения 23; 200/200 выбранных сообщений и страница истории идентичны |
|
||||
| Маркеры: подтверждено до старта дампа / к концу дампа / всего | 76 / 137 / 221 |
|
||||
| Маркеры в восстановленной БД | 80, ровно `1…80` (непрерывный префикс) |
|
||||
|
||||
Восстановленная база — согласованный снимок на момент начала `pg_dump` (маркеры 77–80 закоммичены между чтением счётчика и взятием снимка): ни одной «дырки», но всё, что закоммичено после снимка (141 из 221 маркера, включая 57 подтверждённых клиенту *во время* дампа), в backup отсутствует.
|
||||
|
||||
**Выводы.**
|
||||
- H1 подтверждена: при V1 горячий чат стоит 12 мс и ~3 600 буферов на каждую страницу (вся история чата), при V3 — 0.03 мс и 10–12 буферов независимо от размера чата и глубины cursor; для маленьких чатов V1 уже достаточен — неравномерность распределения и есть причина выбора составного индекса. V3 против V2 убирает Incremental Sort и делает cursor `(created_at, id)` полностью индексным условием ценой +2.1 MiB и +31 байта WAL на вставку.
|
||||
- H2 подтверждена: OFFSET в середине горячего чата — 20–25 мс при любом некрывающем индексе (≈750× медленнее keyset при V3). Покрывающий индекс ускоряет OFFSET до 2 мс, но остаётся O(k) (390 буферов) и стоит +136 % WAL и 4.4× размера индекса — для API выбран keyset + V3, а не покрытие.
|
||||
- H3 подтверждена по WAL/размеру; по CPU различия V1–V3 на этом железе в пределах шума.
|
||||
- H4 подтверждена: полное совпадение после восстановления в отдельный сервер. **RTO** этого набора ≈ 4 с (дамп + восстановление + сверка), растёт примерно линейно с объёмом данных и индексов; к нему добавляются запуск нового сервера и переключение DSN приложения. **RPO** логического backup = интервал между backup: всё после начала последнего `pg_dump` теряется. Уменьшить RPO до секунд можно непрерывной архивацией WAL + base backup (PITR) или репликой; это не реализовано.
|
||||
|
||||
**Ограничения измерений.** Тёплый кэш (весь набор помещается в `shared_buffers`+page cache, `shared read = 0`): измерена CPU/буферная стоимость, а не дисковый I/O; на холодном кэше разрыв V1/V3 для горячего чата будет больше. Один прогон набора с фиксированным seed (UUID случайны); 15 повторов на запрос. В тесте записи нет FK и COMMIT на каждую строку (изолирована стоимость индекса; в API к ней добавляется fsync COMMIT). Планы сняты с литералами; приложение через psycopg после 5 выполнений использует prepared statements — для keyset-запроса generic plan тот же Index Scan, но для сильно неравномерных данных это стоит проверять отдельно (`plan_cache_mode`). Числа получены на нативном стенде; в Compose с лимитами CPU/RAM их нужно повторить (`make research-track2`).
|
||||
|
||||
## Ответы на вопросы защиты
|
||||
|
||||
- **Что подтверждает COMMIT?** Что запись транзакции сброшена в WAL на диск (`synchronous_commit=on`, `fsync=on` — значения по умолчанию, не менялись) и переживёт падение процесса/ОС. При `synchronous_commit=off` или `fsync=off` подтверждённые транзакции могут потеряться; от потери диска/хоста COMMIT на одном сервере не защищает — для этого нужны реплика/backup.
|
||||
- **Где хранятся сессии и что будет после flush/restart Redis?** В PostgreSQL (таблица `sessions`, только SHA-256 токена). Redis — кэш без persistence: после flush/restart кэш пуст, первое обращение читает PostgreSQL и прогревает кэш; ни одна сессия не теряется и отозванная не воскресает.
|
||||
- **Аутентификация vs доступ к чату.** Аутентификация (Bearer → пользователь) — `authenticate()`; авторизация — проверка членства в каждом запросе к чату. Чужой или несуществующий чат — одинаковый 404, порядок «сначала 401, затем 404/400».
|
||||
- **Hash пароля vs быстрый checksum.** KDF (scrypt) специально медленный и memory-hard с солью, чтобы перебор украденной базы был дорогим; SHA-256 быстрый — годится для целостности и для поиска токена с 256 битами случайности (перебор бессмысленен), но не для паролей с низкой энтропией.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Выдерживается: рестарт любых компонентов с сохранёнными volumes; отказ Redis (полная работоспособность через PostgreSQL); отказ PostgreSQL (ограниченный 503, восстановление без перезапуска API). Не выдерживается: потеря host/диска (одна БД, один host), отказ единственного экземпляра API.
|
||||
- Окно устаревания кэша: если Redis был не перезапущен, а только *сетево* недоступен для экземпляра во время logout, отозванная запись живёт в Redis до дообработки outbox (≤1 с после восстановления связи этого экземпляра); экземпляр, сам не видевший ошибки (появится в lab3 с репликами), может прочитать её в это окно, но не дольше `SESSION_CACHE_TTL_SECONDS` (60 с). Это задокументированное учебное упрощение.
|
||||
- Время сессий в кэше сравнивается с часами API, а `expires_at` назначает БД: на одном host часы общие; при разных хостах нужен NTP.
|
||||
- Нет rate limiting входа (429 в контракте факультативен) — следующий шаг вместе с lab7.
|
||||
- `evidence/` собраны на нативном стенде (см. выше); повторить на Compose: `./scripts/collect-evidence.sh`.
|
||||
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
# Учебный стенд lab2: API + PostgreSQL (источник истины) + Redis (кэш сессий).
|
||||
# Запуск: scripts/init-secrets.sh && cp .env.example .env && docker compose --env-file .env up -d --build --wait
|
||||
name: mtusi-lab2
|
||||
|
||||
x-api-common: &api-common
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-lab2-api:local
|
||||
environment: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: postgres
|
||||
SESSION_CACHE: ${SESSION_CACHE:-redis}
|
||||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
DB_POOL_MIN: "1"
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DB_POOL_TIMEOUT_SECONDS: "2"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
CACHE_TIMEOUT_SECONDS: "0.2"
|
||||
REQUEST_TIMEOUT_SECONDS: "10"
|
||||
MIGRATE_ON_START: "true"
|
||||
secrets: [database_url, cache_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_started # Redis — кэш: API стартует и работает и без него
|
||||
networks: [frontend, backend]
|
||||
read_only: true
|
||||
tmpfs: [/tmp]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
|
||||
services:
|
||||
api:
|
||||
<<: *api-common
|
||||
ports: ["127.0.0.1:${API_PORT:-8080}:8080"]
|
||||
|
||||
# Отдельный экземпляр с коротким TTL только для проверки истечения сессий (make test-expiry).
|
||||
api-ttl:
|
||||
<<: *api-common
|
||||
profiles: [ttl]
|
||||
environment:
|
||||
<<: *api-env
|
||||
SESSION_TTL_SECONDS: "2"
|
||||
ports: ["127.0.0.1:${API_TTL_PORT:-8081}:8080"]
|
||||
|
||||
postgres:
|
||||
image: postgres:16.10-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums"
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
- ./backups:/backups # только для трека 5 (pg_dump/pg_restore)
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
networks: [backend]
|
||||
shm_size: 256m
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "2.0", memory: 1G}
|
||||
|
||||
# Redis — кэш сессий без persistence: после restart/flush кэш холодный, сессии остаются в PostgreSQL.
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
umask 077
|
||||
printf 'requirepass %s\n' "$$(cat /run/secrets/redis_password)" > /tmp/redis.conf
|
||||
exec redis-server /tmp/redis.conf --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru
|
||||
user: redis # не root: entrypoint образа не понижает права для команды sh
|
||||
secrets: [redis_password]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
|
||||
# «Новое хранилище» для эксперимента backup → restore (трек 5.2). Отдельный volume.
|
||||
postgres-restore:
|
||||
image: postgres:16.10-alpine
|
||||
profiles: [research]
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: restore_check
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums"
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgrestore:/var/lib/postgresql/data
|
||||
- ./backups:/backups:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d restore_check -h 127.0.0.1"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
networks: [backend]
|
||||
shm_size: 256m
|
||||
|
||||
networks:
|
||||
frontend: {}
|
||||
backend:
|
||||
internal: true # PostgreSQL и Redis недоступны с хоста и из внешней сети
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
pgrestore: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
redis_password:
|
||||
file: ./secrets/redis_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"run": "load.py --duration 20 --concurrency 4 (repeat, with process sampling)",
|
||||
"successful_rps": 627.178,
|
||||
"latency_ms": {
|
||||
"p50": 5.98,
|
||||
"p95": 8.954,
|
||||
"p99": 11.843
|
||||
},
|
||||
"error_ratio": 0.0,
|
||||
"api_process_cpu_cores_avg": 0.99,
|
||||
"api_process_rss_peak_mib": 80.1,
|
||||
"host": "2 CPU, 8027 MiB RAM, no cgroup limits on native stand"
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 2,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.031,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 17950,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 597.716,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.282,
|
||||
"p95": 9.446,
|
||||
"p99": 12.031
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
# 2026-10-26T17:32:39Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2; mem=8027MiB
|
||||
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
|
||||
{
|
||||
"lab": 2,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.031,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 17950,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 597.716,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.282,
|
||||
"p95": 9.446,
|
||||
"p99": 12.031
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
exit=0
|
||||
@@ -0,0 +1,5 @@
|
||||
# 2026-10-26T17:31:55Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2; mem=8027MiB
|
||||
$ make check
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
@@ -0,0 +1,47 @@
|
||||
# 2026-10-26T17:31:49Z native stand: empty database -> API start applies migrations; second run is a no-op
|
||||
$ grep migrations /srv/stand/api.log
|
||||
api.log:{"ts": "2026-10-26T17:31:46Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0002_cache_invalidation_outbox"}
|
||||
api.log:{"ts": "2026-10-26T17:31:46Z", "level": "info", "logger": "messenger.http", "msg": "migrations checked", "applied": ["0001_init", "0002_cache_invalidation_outbox"]}
|
||||
api-ttl.log:{"ts": "2026-10-26T17:24:57Z", "level": "info", "logger": "messenger.http", "msg": "migrations checked", "applied": []}
|
||||
api-ttl.log:{"ts": "2026-10-26T17:31:46Z", "level": "info", "logger": "messenger.http", "msg": "migrations checked", "applied": []}
|
||||
|
||||
$ python -m messenger migrate # repeat
|
||||
{"applied": []}
|
||||
|
||||
$ psql -c "\dt" ; select * from schema_migrations
|
||||
List of relations
|
||||
Schema | Name | Type | Owner
|
||||
--------+-----------------------------+-------+-----------
|
||||
public | chat_members | table | messenger
|
||||
public | chats | table | messenger
|
||||
public | messages | table | messenger
|
||||
public | schema_migrations | table | messenger
|
||||
public | session_cache_invalidations | table | messenger
|
||||
public | sessions | table | messenger
|
||||
public | users | table | messenger
|
||||
(7 rows)
|
||||
|
||||
version | checksum | applied_at
|
||||
--------------------------------+--------------+-------------------------------
|
||||
0001_init | 64c1576d88cc | 2026-10-26 20:31:46.828818+03
|
||||
0002_cache_invalidation_outbox | fcebf1c9345a | 2026-10-26 20:31:46.834237+03
|
||||
(2 rows)
|
||||
|
||||
List of relations
|
||||
Schema | Name | Type | Owner | Table
|
||||
--------+----------------------------------+-------+-----------+-----------------------------
|
||||
public | chat_members_pkey | index | messenger | chat_members
|
||||
public | chat_members_position_key | index | messenger | chat_members
|
||||
public | chat_members_user_idx | index | messenger | chat_members
|
||||
public | chats_pkey | index | messenger | chats
|
||||
public | messages_chat_created_id_idx | index | messenger | messages
|
||||
public | messages_pkey | index | messenger | messages
|
||||
public | schema_migrations_pkey | index | messenger | schema_migrations
|
||||
public | session_cache_invalidations_pkey | index | messenger | session_cache_invalidations
|
||||
public | sessions_expires_idx | index | messenger | sessions
|
||||
public | sessions_pkey | index | messenger | sessions
|
||||
public | sessions_user_idx | index | messenger | sessions
|
||||
public | users_pkey | index | messenger | users
|
||||
public | users_username_key | index | messenger | users
|
||||
(13 rows)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# 2026-10-26T17:38:02Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/dev/restart-native.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/dev/restart-native.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
@@ -0,0 +1 @@
|
||||
19
|
||||
@@ -0,0 +1,14 @@
|
||||
2026-10-26T17:38:04Z before: api=2679 postgres=2735 redis=3086
|
||||
2026-10-26T17:38:07Z after: api=3157 postgres=3141 redis=3151
|
||||
== 2026-10-26T18:10:08Z before restart
|
||||
api status=running started=2026-10-26T18:08:46.692820352Z pid=29494
|
||||
postgres status=running started=2026-10-26T18:05:34.636862226Z pid=25034
|
||||
redis status=running started=2026-10-26T18:05:34.636155192Z pid=25035
|
||||
volume kept: hh_pgdata
|
||||
volume kept: mtusi-lab2_pgdata
|
||||
== 2026-10-26T18:10:15Z after restart
|
||||
api status=running started=2026-10-26T18:10:14.919014494Z pid=31955
|
||||
postgres status=running started=2026-10-26T18:10:12.189358354Z pid=31771
|
||||
redis status=running started=2026-10-26T18:10:12.117424093Z pid=31761
|
||||
volume kept: hh_pgdata
|
||||
volume kept: mtusi-lab2_pgdata
|
||||
@@ -0,0 +1,5 @@
|
||||
# 2026-10-26T17:31:55Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2; mem=8027MiB
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,14 @@
|
||||
# 2026-10-26T17:33:58Z native stand (Docker Hub недоступен из среды сборки, см. REPORT.md)
|
||||
host: Linux 6.18.44-fc-v153 x86_64; Intel(R) Xeon(R) Processor @ 2.80GHz; nproc=2; RAM=8027 MiB
|
||||
python: Python 3.13.17
|
||||
starlette 1.7.0 | uvicorn 0.54.0 | anyio 4.15.1
|
||||
psycopg 3.2.10 (pure-python impl, system libpq 16.15), psycopg_pool 3.2.6, redis-py 5.2.1 — из git-тегов upstream
|
||||
postgresql server: 16.15 (Ubuntu 16.15-0ubuntu0.24.04.1)
|
||||
redis server: v=7.0.15
|
||||
|
||||
processes:
|
||||
1551 05:25 29992 /usr/lib/postgresql/16/bin/postgres -D /srv/stand/pg-restore -p 5433 -k /run/postgresql
|
||||
2499 02:12 81488 /home/claude/venv/bin/python -m messenger serve
|
||||
2679 01:40 81992 /home/claude/venv/bin/python -m messenger serve
|
||||
2716 01:37 7652 redis-server *:6379
|
||||
2735 01:22 30248 /usr/lib/postgresql/16/bin/postgres -D /srv/stand/pg -p 5432 -k /run/postgresql
|
||||
@@ -0,0 +1,23 @@
|
||||
# 2026-10-26T17:31:57Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2; mem=8027MiB
|
||||
$ python3 student-tests/test_api.py
|
||||
## API, auth, validation, ACL, transactions (http://localhost:8080)
|
||||
PASS password stored only as scrypt KDF hash; User has no secret fields 0.36s
|
||||
PASS wrong password and unknown user: identical 401 + Basic challenge 0.80s
|
||||
PASS malformed Basic / Bearer on /auth/sessions / body on login 0.01s
|
||||
PASS password with ':' and non-ASCII UTF-8 works (RFC 7617) 0.57s
|
||||
PASS opaque 256-bit token, unique per login, DB stores only SHA-256 0.66s
|
||||
PASS only Bearer accepted; Basic and X-User-Id never authenticate 0.01s
|
||||
PASS logout revokes only the current session; repeat -> 401 0.92s
|
||||
PASS strict JSON validation -> 400 0.02s
|
||||
PASS message text: 4000 code points OK, 4001 / whitespace / sender_id -> 400 0.02s
|
||||
PASS SQL is parameterized: injection payloads stored verbatim 0.01s
|
||||
PASS ACL: outsider gets 404 everywhere (even for invalid body) 0.02s
|
||||
PASS create chat validation: duplicates, caller missing, unknown member 0.00s
|
||||
PASS compound create-chat rolls back on FK failure (no orphan chat) 0.14s chats before=2, failed insert left 0 rows
|
||||
PASS registration race: 20 parallel same username -> exactly one 201 2.91s 201x1, 409x19, rows=1
|
||||
PASS keyset pagination: 120 msgs, varying limit, no gaps/duplicates 0.54s 4 pages
|
||||
PASS chat list: only own chats, ordered, cursor bound to user 0.64s
|
||||
PASS read-after-write: confirmed message visible to next read 0.01s
|
||||
PASS unknown route 404 / wrong method 405 in error format 0.01s
|
||||
summary: 18 passed, 0 failed
|
||||
exit=0
|
||||
@@ -0,0 +1,13 @@
|
||||
# 2026-10-26T17:35:09Z native stand, Redis v=7.0.15
|
||||
$ CACHE_URL_FILE=... python student-tests/test_cache_race.py
|
||||
PASS read-through put makes entry readable
|
||||
PASS entry TTL capped by SESSION_CACHE_TTL_SECONDS pttl=60000ms
|
||||
PASS invalidate removes entry
|
||||
PASS late put after logout is refused by tombstone
|
||||
PASS tombstone outlives any cache entry pttl=75000ms
|
||||
PASS entry never outlives session expiry pttl=1198ms
|
||||
PASS expired session not served from cache
|
||||
cache unavailable; falling back to PostgreSQL
|
||||
PASS degraded mode: cache is not read until outbox drained
|
||||
summary: all passed
|
||||
exit=0
|
||||
@@ -0,0 +1,8 @@
|
||||
# 2026-10-26T17:32:07Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2; mem=8027MiB
|
||||
$ python3 student-tests/test_expiry.py
|
||||
## session expiry, SESSION_TTL_SECONDS=2 (http://localhost:8081)
|
||||
PASS expires_at = now + SESSION_TTL_SECONDS 0.54s remaining 2.00s
|
||||
PASS token valid before expiry, 401 Bearer after (cache warmed) 3.59s
|
||||
PASS re-login after expiry issues a new valid session 3.79s
|
||||
summary: 3 passed, 0 failed
|
||||
exit=0
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-10-26T17:32:19Z native stand; Linux 6.18.44-fc-v153 x86_64; nproc=2; mem=8027MiB
|
||||
$ python3 student-tests/test_failures.py
|
||||
## dependency failures (http://localhost:8080)
|
||||
PASS Redis down: API bypasses cache via PostgreSQL, no anonymous trust 4.75s fallback latency 0.005s; outbox rows during outage=1, drained after recovery
|
||||
PASS PostgreSQL down: bounded 503 in error format; anonymous still 401 14.31s latency(s)={'ready': 1.5, 'post_message': 2.0, 'history': 2.0, 'login': 2.0, 'register': 2.0}; ready again in 2.5s
|
||||
summary: 2 passed, 0 failed
|
||||
exit=0
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"measured_at": "2026-10-26T17:30:45+00:00",
|
||||
"dump": {
|
||||
"format": "custom (-Fc -Z 6)",
|
||||
"seconds": 0.783,
|
||||
"bytes": 7220447
|
||||
},
|
||||
"restore": {
|
||||
"parallel_jobs": 4,
|
||||
"seconds": 0.969
|
||||
},
|
||||
"verify_seconds": 2.372,
|
||||
"rto_dump_plus_restore_plus_verify_seconds": 4.124,
|
||||
"table_checks": {
|
||||
"users": {
|
||||
"source": "2000",
|
||||
"restored": "2000",
|
||||
"match": true
|
||||
},
|
||||
"chats": {
|
||||
"source": "300",
|
||||
"restored": "300",
|
||||
"match": true
|
||||
},
|
||||
"chat_members": {
|
||||
"source": "7720",
|
||||
"restored": "7720",
|
||||
"match": true
|
||||
},
|
||||
"messages_static": {
|
||||
"source": "120000",
|
||||
"restored": "120000",
|
||||
"match": true
|
||||
},
|
||||
"schema_migrations": {
|
||||
"source": "2",
|
||||
"restored": "2",
|
||||
"match": true
|
||||
},
|
||||
"indexes": {
|
||||
"source": "13",
|
||||
"restored": "13",
|
||||
"match": true
|
||||
},
|
||||
"constraints": {
|
||||
"source": "23",
|
||||
"restored": "23",
|
||||
"match": true
|
||||
}
|
||||
},
|
||||
"sample_messages": {
|
||||
"count": 200,
|
||||
"identical": true
|
||||
},
|
||||
"history_page_hot_chat_offset_1000_identical": true,
|
||||
"concurrent_writes": {
|
||||
"acked_before_dump_started": 76,
|
||||
"acked_when_dump_finished": 137,
|
||||
"acked_total": 221,
|
||||
"present_in_source": 221,
|
||||
"present_in_restored": 80,
|
||||
"restored_is_contiguous_prefix": true,
|
||||
"lost_relative_to_source": 141
|
||||
},
|
||||
"all_match": true
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"generated_at": "2026-10-26T17:30:01+00:00",
|
||||
"params": {
|
||||
"seed": 0.42,
|
||||
"users": 2000,
|
||||
"chats": 300,
|
||||
"messages": 120000,
|
||||
"zipf_s": 1.1
|
||||
},
|
||||
"generation_seconds": 14.58,
|
||||
"messages_total": 120000,
|
||||
"chats_with_messages": 300,
|
||||
"chat_size_top5": [
|
||||
24294,
|
||||
11289,
|
||||
7264,
|
||||
5382,
|
||||
4266
|
||||
],
|
||||
"chat_size_p50": 98,
|
||||
"chat_size_min": 38,
|
||||
"top1_share": 0.2024,
|
||||
"top10_share": 0.5441,
|
||||
"messages_table_mb": 28.3,
|
||||
"server_version": "16.15 (Ubuntu 16.15-0ubuntu0.24.04.1)",
|
||||
"shared_buffers": "128MB"
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
-- btree(chat_id); hot chat, 24294 messages; keyset cursor at 50%
|
||||
Limit (cost=4708.68..4708.80 rows=51 width=213) (actual time=14.248..14.259 rows=51 loops=1)
|
||||
Buffers: shared hit=3642
|
||||
-> Sort (cost=4708.68..4738.72 rows=12019 width=213) (actual time=14.247..14.254 rows=51 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 42kB
|
||||
Buffers: shared hit=3642
|
||||
-> Bitmap Heap Scan on messages (cost=271.12..4307.70 rows=12019 width=213) (actual time=6.934..12.540 rows=12147 loops=1)
|
||||
Recheck Cond: (chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid)
|
||||
Filter: (ROW(created_at, id) > ROW('2026-02-04 13:58:00+03'::timestamp with time zone, 'dc3c4cd9-029a-40dd-976c-64cd71098149'::uuid))
|
||||
Rows Removed by Filter: 12147
|
||||
Heap Blocks: exact=3614
|
||||
Buffers: shared hit=3636
|
||||
-> Bitmap Index Scan on hist_idx (cost=0.00..268.11 rows=23976 width=0) (actual time=1.156..1.156 rows=24294 loops=1)
|
||||
Index Cond: (chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid)
|
||||
Buffers: shared hit=22
|
||||
Planning:
|
||||
Buffers: shared hit=132
|
||||
Planning Time: 0.426 ms
|
||||
Execution Time: 14.305 ms
|
||||
|
||||
-- same chat, OFFSET 12147
|
||||
Limit (cost=5968.35..5968.48 rows=51 width=213) (actual time=24.772..24.782 rows=51 loops=1)
|
||||
Buffers: shared hit=3642, temp read=535 written=676
|
||||
-> Sort (cost=5937.98..5997.92 rows=23976 width=213) (actual time=22.894..24.401 rows=12198 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: external merge Disk: 5392kB
|
||||
Buffers: shared hit=3642, temp read=535 written=676
|
||||
-> Bitmap Heap Scan on messages (cost=274.11..4190.81 rows=23976 width=213) (actual time=1.657..11.046 rows=24294 loops=1)
|
||||
Recheck Cond: (chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid)
|
||||
Heap Blocks: exact=3614
|
||||
Buffers: shared hit=3636
|
||||
-> Bitmap Index Scan on hist_idx (cost=0.00..268.11 rows=23976 width=0) (actual time=1.179..1.179 rows=24294 loops=1)
|
||||
Index Cond: (chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid)
|
||||
Buffers: shared hit=22
|
||||
Planning:
|
||||
Buffers: shared hit=133
|
||||
Planning Time: 0.460 ms
|
||||
Execution Time: 25.426 ms
|
||||
@@ -0,0 +1,29 @@
|
||||
-- btree(chat_id, created_at, id) — выбран в миграции 0001; hot chat, 24294 messages; keyset cursor at 50%
|
||||
Limit (cost=0.42..62.81 rows=51 width=213) (actual time=0.032..0.053 rows=51 loops=1)
|
||||
Buffers: shared hit=12
|
||||
-> Index Scan using hist_idx on messages (cost=0.42..15039.53 rows=12293 width=213) (actual time=0.031..0.049 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid) AND (ROW(created_at, id) > ROW('2026-02-04 13:58:00+03'::timestamp with time zone, 'dc3c4cd9-029a-40dd-976c-64cd71098149'::uuid)))
|
||||
Buffers: shared hit=12
|
||||
Planning:
|
||||
Buffers: shared hit=143
|
||||
Planning Time: 0.506 ms
|
||||
Execution Time: 0.083 ms
|
||||
|
||||
-- same chat, OFFSET 12147
|
||||
Limit (cost=6633.54..6633.67 rows=51 width=213) (actual time=32.900..32.912 rows=51 loops=1)
|
||||
Buffers: shared hit=3796, temp read=535 written=676
|
||||
-> Sort (cost=6603.18..6664.33 rows=24460 width=213) (actual time=29.824..32.370 rows=12198 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: external merge Disk: 5392kB
|
||||
Buffers: shared hit=3796, temp read=535 written=676
|
||||
-> Bitmap Heap Scan on messages (cost=897.98..4820.73 rows=24460 width=213) (actual time=2.958..14.794 rows=24294 loops=1)
|
||||
Recheck Cond: (chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid)
|
||||
Heap Blocks: exact=3614
|
||||
Buffers: shared hit=3790
|
||||
-> Bitmap Index Scan on hist_idx (cost=0.00..891.87 rows=24460 width=0) (actual time=2.442..2.442 rows=24294 loops=1)
|
||||
Index Cond: (chat_id = '0ab807ce-24eb-4da8-8458-ef0836fb3470'::uuid)
|
||||
Buffers: shared hit=176
|
||||
Planning:
|
||||
Buffers: shared hit=144
|
||||
Planning Time: 0.500 ms
|
||||
Execution Time: 33.741 ms
|
||||
@@ -0,0 +1,563 @@
|
||||
{
|
||||
"targets": [
|
||||
{
|
||||
"label": "hot",
|
||||
"chat_id": "0ab807ce-24eb-4da8-8458-ef0836fb3470",
|
||||
"messages": 24294,
|
||||
"offset": 12147,
|
||||
"cursor_ts": "2026-02-04 13:58:00+03",
|
||||
"cursor_id": "dc3c4cd9-029a-40dd-976c-64cd71098149"
|
||||
},
|
||||
{
|
||||
"label": "median",
|
||||
"chat_id": "5455628c-0b94-4d4a-95bf-74145b899408",
|
||||
"messages": 99,
|
||||
"offset": 49,
|
||||
"cursor_ts": "2026-02-04 03:36:40+03",
|
||||
"cursor_id": "01b51bb5-cfb2-4fce-a665-02087836dd8c"
|
||||
},
|
||||
{
|
||||
"label": "small",
|
||||
"chat_id": "74d7b4ab-e4ce-4065-b97b-1b00a2383364",
|
||||
"messages": 38,
|
||||
"offset": 19,
|
||||
"cursor_ts": "2026-02-04 08:40:15+03",
|
||||
"cursor_id": "b94bd4dd-e28c-4bce-8065-8a07a85a72bd"
|
||||
}
|
||||
],
|
||||
"repeats": 15,
|
||||
"variants": {
|
||||
"V0_no_index": {
|
||||
"note": "только PK(id): нет индекса по chat_id",
|
||||
"index_bytes": 0,
|
||||
"queries": {
|
||||
"hot/keyset_first": {
|
||||
"exec_ms_median": 18.126,
|
||||
"exec_ms_p90": 19.806,
|
||||
"shared_hit": 3705,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Gather Merge",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"hot/keyset_middle": {
|
||||
"exec_ms_median": 18.343,
|
||||
"exec_ms_p90": 22.394,
|
||||
"shared_hit": 3647,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Gather Merge",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"hot/offset_middle": {
|
||||
"exec_ms_median": 28.459,
|
||||
"exec_ms_p90": 33.177,
|
||||
"shared_hit": 3617,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"median/keyset_first": {
|
||||
"exec_ms_median": 14.429,
|
||||
"exec_ms_p90": 15.984,
|
||||
"shared_hit": 3617,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"median/keyset_middle": {
|
||||
"exec_ms_median": 15.644,
|
||||
"exec_ms_p90": 16.287,
|
||||
"shared_hit": 3647,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Gather Merge",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"median/offset_middle": {
|
||||
"exec_ms_median": 14.285,
|
||||
"exec_ms_p90": 15.051,
|
||||
"shared_hit": 3617,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"small/keyset_first": {
|
||||
"exec_ms_median": 13.895,
|
||||
"exec_ms_p90": 17.684,
|
||||
"shared_hit": 3617,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"small/keyset_middle": {
|
||||
"exec_ms_median": 17.617,
|
||||
"exec_ms_p90": 19.349,
|
||||
"shared_hit": 3647,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Gather Merge",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
},
|
||||
"small/offset_middle": {
|
||||
"exec_ms_median": 14.198,
|
||||
"exec_ms_p90": 16.743,
|
||||
"shared_hit": 3617,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Seq Scan"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"V1_chat": {
|
||||
"note": "btree(chat_id)",
|
||||
"index_bytes": 876544,
|
||||
"queries": {
|
||||
"hot/keyset_first": {
|
||||
"exec_ms_median": 12.668,
|
||||
"exec_ms_p90": 15.04,
|
||||
"shared_hit": 3636,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/keyset_middle": {
|
||||
"exec_ms_median": 12.078,
|
||||
"exec_ms_p90": 13.361,
|
||||
"shared_hit": 3636,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/offset_middle": {
|
||||
"exec_ms_median": 20.116,
|
||||
"exec_ms_p90": 26.066,
|
||||
"shared_hit": 3636,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_first": {
|
||||
"exec_ms_median": 0.163,
|
||||
"exec_ms_p90": 0.192,
|
||||
"shared_hit": 99,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_middle": {
|
||||
"exec_ms_median": 0.081,
|
||||
"exec_ms_p90": 0.109,
|
||||
"shared_hit": 99,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/offset_middle": {
|
||||
"exec_ms_median": 0.075,
|
||||
"exec_ms_p90": 0.105,
|
||||
"shared_hit": 99,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_first": {
|
||||
"exec_ms_median": 0.056,
|
||||
"exec_ms_p90": 0.061,
|
||||
"shared_hit": 40,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_middle": {
|
||||
"exec_ms_median": 0.038,
|
||||
"exec_ms_p90": 0.041,
|
||||
"shared_hit": 40,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/offset_middle": {
|
||||
"exec_ms_median": 0.038,
|
||||
"exec_ms_p90": 0.041,
|
||||
"shared_hit": 40,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"V2_chat_created": {
|
||||
"note": "btree(chat_id, created_at)",
|
||||
"index_bytes": 4898816,
|
||||
"queries": {
|
||||
"hot/keyset_first": {
|
||||
"exec_ms_median": 0.036,
|
||||
"exec_ms_p90": 0.061,
|
||||
"shared_hit": 10,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Incremental Sort",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/keyset_middle": {
|
||||
"exec_ms_median": 0.038,
|
||||
"exec_ms_p90": 0.049,
|
||||
"shared_hit": 11,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Incremental Sort",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/offset_middle": {
|
||||
"exec_ms_median": 21.445,
|
||||
"exec_ms_p90": 25.194,
|
||||
"shared_hit": 3737,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_first": {
|
||||
"exec_ms_median": 0.048,
|
||||
"exec_ms_p90": 0.052,
|
||||
"shared_hit": 54,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Incremental Sort",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_middle": {
|
||||
"exec_ms_median": 0.07,
|
||||
"exec_ms_p90": 0.102,
|
||||
"shared_hit": 53,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/offset_middle": {
|
||||
"exec_ms_median": 0.135,
|
||||
"exec_ms_p90": 0.16,
|
||||
"shared_hit": 100,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_first": {
|
||||
"exec_ms_median": 0.054,
|
||||
"exec_ms_p90": 0.082,
|
||||
"shared_hit": 41,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Incremental Sort",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_middle": {
|
||||
"exec_ms_median": 0.048,
|
||||
"exec_ms_p90": 0.056,
|
||||
"shared_hit": 23,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/offset_middle": {
|
||||
"exec_ms_median": 0.05,
|
||||
"exec_ms_p90": 0.059,
|
||||
"shared_hit": 41,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Incremental Sort",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"V3_chat_created_id": {
|
||||
"note": "btree(chat_id, created_at, id) — выбран в миграции 0001",
|
||||
"index_bytes": 7094272,
|
||||
"queries": {
|
||||
"hot/keyset_first": {
|
||||
"exec_ms_median": 0.023,
|
||||
"exec_ms_p90": 0.034,
|
||||
"shared_hit": 10,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/keyset_middle": {
|
||||
"exec_ms_median": 0.033,
|
||||
"exec_ms_p90": 0.038,
|
||||
"shared_hit": 12,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/offset_middle": {
|
||||
"exec_ms_median": 25.014,
|
||||
"exec_ms_p90": 28.762,
|
||||
"shared_hit": 3790,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_first": {
|
||||
"exec_ms_median": 0.028,
|
||||
"exec_ms_p90": 0.036,
|
||||
"shared_hit": 53,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_middle": {
|
||||
"exec_ms_median": 0.05,
|
||||
"exec_ms_p90": 0.077,
|
||||
"shared_hit": 52,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/offset_middle": {
|
||||
"exec_ms_median": 0.128,
|
||||
"exec_ms_p90": 0.133,
|
||||
"shared_hit": 100,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_first": {
|
||||
"exec_ms_median": 0.026,
|
||||
"exec_ms_p90": 0.03,
|
||||
"shared_hit": 42,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_middle": {
|
||||
"exec_ms_median": 0.028,
|
||||
"exec_ms_p90": 0.029,
|
||||
"shared_hit": 22,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Sort",
|
||||
"Bitmap Heap Scan",
|
||||
"Bitmap Index Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/offset_middle": {
|
||||
"exec_ms_median": 0.023,
|
||||
"exec_ms_p90": 0.038,
|
||||
"shared_hit": 42,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Scan using hist_idx"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"V4_covering": {
|
||||
"note": "covering: + INCLUDE(sender_id, text)",
|
||||
"index_bytes": 31449088,
|
||||
"queries": {
|
||||
"hot/keyset_first": {
|
||||
"exec_ms_median": 0.031,
|
||||
"exec_ms_p90": 0.033,
|
||||
"shared_hit": 5,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/keyset_middle": {
|
||||
"exec_ms_median": 0.035,
|
||||
"exec_ms_p90": 0.036,
|
||||
"shared_hit": 6,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"hot/offset_middle": {
|
||||
"exec_ms_median": 2.14,
|
||||
"exec_ms_p90": 2.4,
|
||||
"shared_hit": 390,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_first": {
|
||||
"exec_ms_median": 0.032,
|
||||
"exec_ms_p90": 0.052,
|
||||
"shared_hit": 6,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/keyset_middle": {
|
||||
"exec_ms_median": 0.034,
|
||||
"exec_ms_p90": 0.035,
|
||||
"shared_hit": 5,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"median/offset_middle": {
|
||||
"exec_ms_median": 0.042,
|
||||
"exec_ms_p90": 0.044,
|
||||
"shared_hit": 7,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_first": {
|
||||
"exec_ms_median": 0.027,
|
||||
"exec_ms_p90": 0.029,
|
||||
"shared_hit": 5,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/keyset_middle": {
|
||||
"exec_ms_median": 0.022,
|
||||
"exec_ms_p90": 0.022,
|
||||
"shared_hit": 4,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
},
|
||||
"small/offset_middle": {
|
||||
"exec_ms_median": 0.025,
|
||||
"exec_ms_p90": 0.025,
|
||||
"shared_hit": 5,
|
||||
"shared_read": 0,
|
||||
"plan": [
|
||||
"Limit",
|
||||
"Index Only Scan using hist_idx"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"rows_per_run": 50000,
|
||||
"runs": 5,
|
||||
"note": "один INSERT на строку внутри одной транзакции (без fsync на COMMIT); FK исключён, чтобы изолировать стоимость поддержки индекса истории",
|
||||
"variants": {
|
||||
"V0_no_index": {
|
||||
"note": "только PK(id): нет индекса по chat_id",
|
||||
"ms_median": 640.3,
|
||||
"us_per_insert": 12.81,
|
||||
"wal_bytes_per_insert": 360.4,
|
||||
"index_bytes_after_run": 0,
|
||||
"runs_ms": [
|
||||
533.5,
|
||||
646.4,
|
||||
640.3,
|
||||
641.4,
|
||||
612.8
|
||||
],
|
||||
"overhead_vs_no_index_pct": 0.0
|
||||
},
|
||||
"V1_chat": {
|
||||
"note": "btree(chat_id)",
|
||||
"ms_median": 778.0,
|
||||
"us_per_insert": 15.56,
|
||||
"wal_bytes_per_insert": 438.4,
|
||||
"index_bytes_after_run": 458752,
|
||||
"runs_ms": [
|
||||
778.0,
|
||||
995.6,
|
||||
826.4,
|
||||
660.6,
|
||||
758.6
|
||||
],
|
||||
"overhead_vs_no_index_pct": 21.5
|
||||
},
|
||||
"V2_chat_created": {
|
||||
"note": "btree(chat_id, created_at)",
|
||||
"ms_median": 772.0,
|
||||
"us_per_insert": 15.44,
|
||||
"wal_bytes_per_insert": 471.0,
|
||||
"index_bytes_after_run": 3284992,
|
||||
"runs_ms": [
|
||||
805.3,
|
||||
772.0,
|
||||
756.2,
|
||||
757.3,
|
||||
817.3
|
||||
],
|
||||
"overhead_vs_no_index_pct": 20.5
|
||||
},
|
||||
"V3_chat_created_id": {
|
||||
"note": "btree(chat_id, created_at, id) — выбран в миграции 0001",
|
||||
"ms_median": 744.0,
|
||||
"us_per_insert": 14.88,
|
||||
"wal_bytes_per_insert": 501.8,
|
||||
"index_bytes_after_run": 4743168,
|
||||
"runs_ms": [
|
||||
763.3,
|
||||
676.1,
|
||||
744.0,
|
||||
749.5,
|
||||
707.4
|
||||
],
|
||||
"overhead_vs_no_index_pct": 16.2
|
||||
},
|
||||
"V4_covering": {
|
||||
"note": "covering: + INCLUDE(sender_id, text)",
|
||||
"ms_median": 928.8,
|
||||
"us_per_insert": 18.58,
|
||||
"wal_bytes_per_insert": 849.2,
|
||||
"index_bytes_after_run": 21970944,
|
||||
"runs_ms": [
|
||||
928.8,
|
||||
973.1,
|
||||
970.6,
|
||||
872.0,
|
||||
857.7
|
||||
],
|
||||
"overhead_vs_no_index_pct": 45.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
-- 0001: базовая схема lab2. Источник истины — PostgreSQL.
|
||||
-- Миграция выполняется раннером в одной транзакции и регистрируется в schema_migrations.
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_username_key UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
CONSTRAINT users_password_hash_kdf CHECK (password_hash LIKE '$scrypt$%')
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title text NOT NULL,
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
-- Состав участников неизменен в обязательном API; position сохраняет порядок member_ids из запроса.
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position),
|
||||
CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99)
|
||||
);
|
||||
-- GET /chats: «чаты пользователя» — поиск по user_id.
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
-- Отправитель обязан быть участником этого чата: инвариант защищён на уровне БД.
|
||||
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
);
|
||||
-- История: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n.
|
||||
-- Обоснование и EXPLAIN — REPORT.md и evidence/track2/.
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Серверные сессии. Хранится только SHA-256 от токена (токен — 256 бит случайности),
|
||||
-- поэтому утечка дампа не даёт готовых Bearer-токенов.
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32),
|
||||
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- 0002: транзакционный outbox инвалидации кэша сессий (уровень 4).
|
||||
-- Строка пишется в той же транзакции, что и revoked_at. Если Redis недоступен в момент logout,
|
||||
-- фоновый процесс API повторит инвалидацию после восстановления Redis и только затем снова
|
||||
-- начнёт читать сессии из кэша.
|
||||
CREATE TABLE session_cache_invalidations (
|
||||
token_hash bytea PRIMARY KEY REFERENCES sessions (token_hash) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
# Runtime-зависимости API (точные версии, проверенные на стенде; см. REPORT.md).
|
||||
starlette==1.7.0
|
||||
anyio==4.15.1
|
||||
idna==3.20
|
||||
typing_extensions==4.16.0
|
||||
uvicorn==0.54.0
|
||||
h11==0.16.0
|
||||
click==8.5.0
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-binary==3.2.10
|
||||
psycopg-pool==3.2.6
|
||||
redis==5.2.1
|
||||
@@ -0,0 +1,57 @@
|
||||
-- Синтетический набор для трека 5.2 (запускается research/track2.py в отдельной БД research).
|
||||
-- Переменные psql: :seed, :users, :chats, :messages, :zipf_s.
|
||||
-- Размеры чатов неравномерны: доля чата ранга k пропорциональна 1/k^s (закон Ципфа).
|
||||
-- Сообщения вставляются в глобальном хронологическом порядке, чаты перемешаны во времени —
|
||||
-- как в реальном мессенджере, поэтому строки большого чата разбросаны по всей heap.
|
||||
\set ON_ERROR_STOP on
|
||||
SELECT setseed(:seed);
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
SELECT gen_random_uuid(), 'r_' || g, 'Research user ' || g,
|
||||
'$scrypt$ln=15,r=8,p=3$synthetic$synthetic', timestamptz '2026-01-01 00:00:00Z' + g * interval '1 second'
|
||||
FROM generate_series(1, :users) g;
|
||||
|
||||
CREATE TEMP TABLE r_users AS SELECT row_number() OVER (ORDER BY username) AS n, id FROM users;
|
||||
CREATE TEMP TABLE r_chats AS SELECT g AS rank, gen_random_uuid() AS id FROM generate_series(1, :chats) g;
|
||||
|
||||
-- created_by временно = первый пользователь, ниже заменяется на участника с position = 0.
|
||||
INSERT INTO chats (id, title, created_by, created_at)
|
||||
SELECT c.id, 'research chat #' || c.rank, (SELECT id FROM r_users WHERE n = 1),
|
||||
timestamptz '2026-01-02 00:00:00Z' + c.rank * interval '1 second'
|
||||
FROM r_chats c;
|
||||
|
||||
-- От 2 до 50 участников; LATERAL коррелирован с c.rank, поэтому random() вычисляется для каждого чата.
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT c.id, u.id, (row_number() OVER (PARTITION BY c.id ORDER BY u.n) - 1)::smallint
|
||||
FROM r_chats c
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT DISTINCT 1 + floor(random() * :users)::int AS n
|
||||
FROM generate_series(1, 2 + (c.rank * 7919) % 49)) pick
|
||||
JOIN r_users u USING (n);
|
||||
|
||||
UPDATE chats SET created_by = m.user_id
|
||||
FROM chat_members m WHERE m.chat_id = chats.id AND m.position = 0;
|
||||
|
||||
CREATE TEMP TABLE r_w AS SELECT rank, id, 1.0 / power(rank, :zipf_s) AS w FROM r_chats;
|
||||
CREATE TEMP TABLE r_cum AS
|
||||
SELECT rank, id,
|
||||
coalesce(sum(w) OVER (ORDER BY rank ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0) / sum(w) OVER () AS lo,
|
||||
sum(w) OVER (ORDER BY rank) / sum(w) OVER () AS hi
|
||||
FROM r_w;
|
||||
CREATE TEMP TABLE r_mem AS SELECT chat_id, array_agg(user_id ORDER BY position) AS members
|
||||
FROM chat_members GROUP BY chat_id;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id,
|
||||
m.members[1 + floor(random() * cardinality(m.members))::int],
|
||||
'synthetic message ' || x.g || repeat(' lorem ipsum', (x.g % 23)::int),
|
||||
timestamptz '2026-02-01 00:00:00Z' + x.g * interval '5 seconds'
|
||||
FROM (SELECT g, random() AS r FROM generate_series(1, :messages) g) x
|
||||
JOIN r_cum c ON x.r >= c.lo AND x.r < c.hi
|
||||
JOIN r_mem m ON m.chat_id = c.id
|
||||
ORDER BY x.g;
|
||||
|
||||
VACUUM (ANALYZE) users;
|
||||
VACUUM (ANALYZE) chats;
|
||||
VACUUM (ANALYZE) chat_members;
|
||||
VACUUM (ANALYZE) messages;
|
||||
@@ -0,0 +1,430 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 5.2: план запроса истории и восстановление из backup. Stdlib, Python 3.9+.
|
||||
|
||||
Шаги (по умолчанию все): setup → generate → plans → writes → backup.
|
||||
Работает в ОТДЕЛЬНОЙ БД `research` (рабочая БД messenger не меняется). Команды доступа к
|
||||
PostgreSQL задаются переменными окружения (по умолчанию — Compose этого репозитория):
|
||||
|
||||
SRC_PSQL psql к исходному серверу (docker compose ... exec -T postgres psql -U messenger)
|
||||
SRC_PG_DUMP pg_dump на исходном сервере (docker compose ... exec -T postgres pg_dump -U messenger)
|
||||
DST_PSQL psql к НОВОМУ серверу (docker compose ... --profile research exec -T postgres-restore psql -U messenger)
|
||||
DST_PG_RESTORE pg_restore на новом сервере (... exec -T postgres-restore pg_restore -U messenger)
|
||||
DUMP_PATH путь файла дампа для pg_dump/pg_restore (/backups/research.dump, bind mount ./backups)
|
||||
DUMP_HOST_PATH тот же файл с точки зрения хоста (backups/research.dump) — для размера
|
||||
|
||||
Результаты: evidence/track2/*.json, *.txt. Секретов и токенов не содержат.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "track2"
|
||||
DC = os.environ.get("COMPOSE", "docker compose --env-file .env")
|
||||
SRC_PSQL = os.environ.get("SRC_PSQL", DC + " exec -T postgres psql -U messenger")
|
||||
SRC_PG_DUMP = os.environ.get("SRC_PG_DUMP", DC + " exec -T postgres pg_dump -U messenger")
|
||||
DST_PSQL = os.environ.get("DST_PSQL", DC + " --profile research exec -T postgres-restore psql -U messenger")
|
||||
DST_PG_RESTORE = os.environ.get("DST_PG_RESTORE", DC + " --profile research exec -T postgres-restore pg_restore -U messenger")
|
||||
DUMP_PATH = os.environ.get("DUMP_PATH", "/backups/research.dump")
|
||||
DUMP_HOST_PATH = Path(os.environ.get("DUMP_HOST_PATH", str(ROOT / "backups" / "research.dump")))
|
||||
DB = "research"
|
||||
RESTORED_DB = "research_restored"
|
||||
|
||||
VARIANTS = {
|
||||
# имя: (DDL индекса истории или None, комментарий)
|
||||
"V0_no_index": (None, "только PK(id): нет индекса по chat_id"),
|
||||
"V1_chat": ("CREATE INDEX hist_idx ON messages (chat_id)", "btree(chat_id)"),
|
||||
"V2_chat_created": ("CREATE INDEX hist_idx ON messages (chat_id, created_at)", "btree(chat_id, created_at)"),
|
||||
"V3_chat_created_id": ("CREATE INDEX hist_idx ON messages (chat_id, created_at, id)",
|
||||
"btree(chat_id, created_at, id) — выбран в миграции 0001"),
|
||||
"V4_covering": ("CREATE INDEX hist_idx ON messages (chat_id, created_at, id) INCLUDE (sender_id, text)",
|
||||
"covering: + INCLUDE(sender_id, text)"),
|
||||
}
|
||||
PRODUCTION_INDEX = "messages_chat_created_id_idx"
|
||||
|
||||
|
||||
def run(prefix, args=(), sql=None, check=True, timeout=1800):
|
||||
proc = subprocess.run(shlex.split(prefix) + list(args), input=sql, capture_output=True, text=True,
|
||||
cwd=ROOT, timeout=timeout)
|
||||
if check and proc.returncode != 0:
|
||||
raise SystemExit("command failed (%s): %s" % (shlex.split(prefix)[0], proc.stderr.strip()[-2000:]))
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def psql(sql, db=DB, prefix=SRC_PSQL, timeout=1800):
|
||||
return run(prefix, ["-X", "-q", "-At", "-v", "ON_ERROR_STOP=1", "-d", db], sql=sql, timeout=timeout)
|
||||
|
||||
|
||||
def one(sql, db=DB, prefix=SRC_PSQL):
|
||||
return psql(sql, db, prefix).strip()
|
||||
|
||||
|
||||
def save(name, data):
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
path = OUT / name
|
||||
if isinstance(data, str):
|
||||
path.write_text(data)
|
||||
else:
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
print(" saved", path.relative_to(ROOT))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
def step_setup(args):
|
||||
print("[setup] recreate database %s and apply migrations" % DB)
|
||||
psql("DROP DATABASE IF EXISTS %s WITH (FORCE);" % DB, db="postgres")
|
||||
psql("CREATE DATABASE %s;" % DB, db="postgres")
|
||||
# Та же схема и тот же учёт версий, что у раннера приложения (src/messenger/db.py).
|
||||
psql("CREATE TABLE schema_migrations (version text PRIMARY KEY, checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT clock_timestamp());")
|
||||
for path in sorted((ROOT / "migrations").glob("[0-9][0-9][0-9][0-9]_*.sql")):
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
digest = hashlib.sha256(sql.encode()).hexdigest()
|
||||
psql("BEGIN;\n" + sql + "\nINSERT INTO schema_migrations (version, checksum) VALUES ('%s', '%s');\nCOMMIT;"
|
||||
% (path.stem, digest))
|
||||
print(" migrations:", [p.stem for p in sorted((ROOT / "migrations").glob("*.sql"))])
|
||||
|
||||
|
||||
def step_generate(args):
|
||||
print("[generate] users=%d chats=%d messages=%d zipf_s=%.2f" % (args.users, args.chats, args.messages, args.zipf_s))
|
||||
sql = (ROOT / "research" / "generate.sql").read_text()
|
||||
t = time.monotonic()
|
||||
run(SRC_PSQL, ["-X", "-q", "-v", "ON_ERROR_STOP=1", "-d", DB, "-v", "seed=%s" % args.seed,
|
||||
"-v", "users=%d" % args.users, "-v", "chats=%d" % args.chats,
|
||||
"-v", "messages=%d" % args.messages, "-v", "zipf_s=%s" % args.zipf_s], sql=sql)
|
||||
took = time.monotonic() - t
|
||||
sizes = [int(x) for x in psql("SELECT count(*) FROM messages GROUP BY chat_id ORDER BY 1 DESC").split()]
|
||||
info = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"params": {"seed": args.seed, "users": args.users, "chats": args.chats, "messages": args.messages,
|
||||
"zipf_s": args.zipf_s},
|
||||
"generation_seconds": round(took, 2),
|
||||
"messages_total": sum(sizes),
|
||||
"chats_with_messages": len(sizes),
|
||||
"chat_size_top5": sizes[:5],
|
||||
"chat_size_p50": int(statistics.median(sizes)),
|
||||
"chat_size_min": sizes[-1],
|
||||
"top1_share": round(sizes[0] / sum(sizes), 4),
|
||||
"top10_share": round(sum(sizes[:10]) / sum(sizes), 4),
|
||||
"messages_table_mb": round(int(one("SELECT pg_table_size('messages')")) / 2**20, 1),
|
||||
"server_version": one("SHOW server_version"),
|
||||
"shared_buffers": one("SHOW shared_buffers"),
|
||||
}
|
||||
save("dataset.json", info)
|
||||
print(" ", {k: info[k] for k in ("messages_total", "chat_size_top5", "chat_size_p50", "chat_size_min")})
|
||||
|
||||
|
||||
def _targets():
|
||||
"""Горячий (крупнейший), медианный и самый маленький чат + курсор/offset на середине истории."""
|
||||
rows = psql("""
|
||||
WITH s AS (SELECT chat_id, count(*) n FROM messages GROUP BY chat_id),
|
||||
r AS (SELECT chat_id, n, row_number() OVER (ORDER BY n DESC, chat_id) rk, count(*) OVER () total FROM s)
|
||||
SELECT 'hot', chat_id, n FROM r WHERE rk = 1
|
||||
UNION ALL SELECT 'median', chat_id, n FROM r WHERE rk = (total + 1) / 2
|
||||
UNION ALL SELECT 'small', chat_id, n FROM r WHERE rk = total;""").splitlines()
|
||||
targets = []
|
||||
for line in rows:
|
||||
label, chat_id, n = line.split("|")
|
||||
n = int(n)
|
||||
offset = n // 2
|
||||
cur = one("SELECT created_at::text || '|' || id FROM messages WHERE chat_id = '%s' "
|
||||
"ORDER BY created_at, id OFFSET %d LIMIT 1" % (chat_id, max(offset - 1, 0)))
|
||||
ts, mid = cur.split("|")
|
||||
targets.append({"label": label, "chat_id": chat_id, "messages": n, "offset": offset, "cursor_ts": ts,
|
||||
"cursor_id": mid})
|
||||
return targets
|
||||
|
||||
|
||||
def _queries(t):
|
||||
c = t["chat_id"]
|
||||
return {
|
||||
"keyset_first": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' "
|
||||
"ORDER BY created_at, id LIMIT 51" % c,
|
||||
"keyset_middle": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' "
|
||||
"AND (created_at, id) > ('%s', '%s') ORDER BY created_at, id LIMIT 51"
|
||||
% (c, t["cursor_ts"], t["cursor_id"]),
|
||||
"offset_middle": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' "
|
||||
"ORDER BY created_at, id OFFSET %d LIMIT 51" % (c, t["offset"]),
|
||||
}
|
||||
|
||||
|
||||
def _plan_summary(plan):
|
||||
nodes = []
|
||||
|
||||
def walk(p):
|
||||
name = p["Node Type"] + (" using " + p["Index Name"] if "Index Name" in p else "")
|
||||
nodes.append(name)
|
||||
for child in p.get("Plans", []):
|
||||
walk(child)
|
||||
walk(plan["Plan"])
|
||||
return nodes
|
||||
|
||||
|
||||
def _apply_variant(name):
|
||||
ddl = VARIANTS[name][0]
|
||||
psql("DROP INDEX IF EXISTS hist_idx; DROP INDEX IF EXISTS %s;" % PRODUCTION_INDEX)
|
||||
if ddl:
|
||||
psql(ddl + ";")
|
||||
psql("VACUUM (ANALYZE) messages;")
|
||||
|
||||
|
||||
def step_plans(args):
|
||||
print("[plans] variants=%s, repeats=%d" % (list(VARIANTS), args.repeats))
|
||||
targets = _targets()
|
||||
results = {"targets": targets, "repeats": args.repeats, "variants": {}}
|
||||
for name, (ddl, note) in VARIANTS.items():
|
||||
_apply_variant(name)
|
||||
size = int(one("SELECT coalesce(pg_relation_size(to_regclass('hist_idx')), 0)"))
|
||||
script, keys = [], []
|
||||
for t in targets:
|
||||
for qname, q in _queries(t).items():
|
||||
for i in range(args.repeats + 1): # первый прогон — прогрев кэша
|
||||
script.append("\\echo @@@ %s %s %d" % (t["label"], qname, i))
|
||||
script.append("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + q + ";")
|
||||
keys.append((t["label"], qname))
|
||||
out = psql("\n".join(script))
|
||||
parsed = {}
|
||||
for block in out.split("@@@ ")[1:]:
|
||||
header, _, body = block.partition("\n")
|
||||
label, qname, i = header.split()
|
||||
if int(i) == 0:
|
||||
continue
|
||||
plan = json.loads(body)[0]
|
||||
parsed.setdefault((label, qname), []).append(plan)
|
||||
variant = {"note": note, "index_bytes": size, "queries": {}}
|
||||
for key in keys:
|
||||
plans = parsed[key]
|
||||
times = [p["Execution Time"] for p in plans]
|
||||
last = plans[-1]["Plan"]
|
||||
variant["queries"]["%s/%s" % key] = {
|
||||
"exec_ms_median": round(statistics.median(times), 3),
|
||||
"exec_ms_p90": round(sorted(times)[int(0.9 * (len(times) - 1))], 3),
|
||||
"shared_hit": last.get("Shared Hit Blocks", 0), "shared_read": last.get("Shared Read Blocks", 0),
|
||||
"plan": _plan_summary(plans[-1]),
|
||||
}
|
||||
results["variants"][name] = variant
|
||||
print(" %-20s index=%7.1f MiB hot/keyset_middle=%8.3f ms hot/offset_middle=%8.3f ms" % (
|
||||
name, size / 2**20, variant["queries"]["hot/keyset_middle"]["exec_ms_median"],
|
||||
variant["queries"]["hot/offset_middle"]["exec_ms_median"]))
|
||||
if name in ("V1_chat", "V3_chat_created_id"):
|
||||
hot = targets[0]
|
||||
text = psql("EXPLAIN (ANALYZE, BUFFERS) " + _queries(hot)["keyset_middle"] + ";")
|
||||
text2 = psql("EXPLAIN (ANALYZE, BUFFERS) " + _queries(hot)["offset_middle"] + ";")
|
||||
save("explain_%s_hot.txt" % name, "-- %s; hot chat, %d messages; keyset cursor at 50%%\n%s\n"
|
||||
"-- same chat, OFFSET %d\n%s" % (note, hot["messages"], text, hot["offset"], text2))
|
||||
_apply_variant("V3_chat_created_id")
|
||||
psql("ALTER INDEX hist_idx RENAME TO %s;" % PRODUCTION_INDEX)
|
||||
save("plans.json", results)
|
||||
|
||||
|
||||
WRITE_FN = r"""
|
||||
DROP TABLE IF EXISTS bench_messages;
|
||||
CREATE TABLE bench_messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(), chat_id uuid NOT NULL, sender_id uuid NOT NULL,
|
||||
text text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp());
|
||||
CREATE OR REPLACE FUNCTION bench_write(n int) RETURNS TABLE(ms double precision, wal_bytes numeric) LANGUAGE plpgsql AS $$
|
||||
DECLARE t0 timestamptz; l0 pg_lsn; pick uuid[]; s uuid;
|
||||
BEGIN
|
||||
-- Чаты выбираются с тем же распределением, что и в истории (выборка chat_id из messages).
|
||||
SELECT array_agg(chat_id) INTO pick FROM (SELECT chat_id FROM messages ORDER BY md5(id::text) LIMIT 5000) x;
|
||||
SELECT id INTO s FROM users LIMIT 1;
|
||||
l0 := pg_current_wal_insert_lsn(); t0 := clock_timestamp();
|
||||
FOR i IN 1..n LOOP
|
||||
INSERT INTO bench_messages (chat_id, sender_id, text)
|
||||
VALUES (pick[1 + (i * 7919) % cardinality(pick)], s, 'bench message ' || i || repeat(' lorem ipsum', i % 23));
|
||||
END LOOP;
|
||||
ms := extract(epoch FROM clock_timestamp() - t0) * 1000;
|
||||
wal_bytes := pg_current_wal_insert_lsn() - l0;
|
||||
RETURN NEXT;
|
||||
END $$;
|
||||
"""
|
||||
|
||||
|
||||
def step_writes(args):
|
||||
print("[writes] %d single-row INSERT per run, %d runs per variant" % (args.write_rows, args.write_runs))
|
||||
psql(WRITE_FN)
|
||||
results = {"rows_per_run": args.write_rows, "runs": args.write_runs,
|
||||
"note": "один INSERT на строку внутри одной транзакции (без fsync на COMMIT); FK исключён, "
|
||||
"чтобы изолировать стоимость поддержки индекса истории", "variants": {}}
|
||||
for name, (ddl, note) in VARIANTS.items():
|
||||
psql("DROP INDEX IF EXISTS bench_hist_idx;")
|
||||
if ddl:
|
||||
psql(ddl.replace("hist_idx ON messages", "bench_hist_idx ON bench_messages") + ";")
|
||||
runs = []
|
||||
for _ in range(args.write_runs):
|
||||
psql("TRUNCATE bench_messages; CHECKPOINT;")
|
||||
ms, wal = one("SELECT ms || '|' || wal_bytes FROM bench_write(%d)" % args.write_rows).split("|")
|
||||
runs.append((float(ms), float(wal)))
|
||||
idx = int(one("SELECT coalesce(pg_relation_size(to_regclass('bench_hist_idx')), 0)"))
|
||||
ms_med = statistics.median(r[0] for r in runs)
|
||||
wal_med = statistics.median(r[1] for r in runs)
|
||||
results["variants"][name] = {
|
||||
"note": note, "ms_median": round(ms_med, 1),
|
||||
"us_per_insert": round(ms_med * 1000 / args.write_rows, 2),
|
||||
"wal_bytes_per_insert": round(wal_med / args.write_rows, 1),
|
||||
"index_bytes_after_run": idx, "runs_ms": [round(r[0], 1) for r in runs]}
|
||||
print(" %-20s %7.2f us/insert %7.1f WAL B/insert index %6.2f MiB" % (
|
||||
name, ms_med * 1000 / args.write_rows, wal_med / args.write_rows, idx / 2**20))
|
||||
psql("DROP TABLE bench_messages; DROP FUNCTION bench_write(int);")
|
||||
base = results["variants"]["V0_no_index"]
|
||||
for v in results["variants"].values():
|
||||
v["overhead_vs_no_index_pct"] = round(100 * (v["us_per_insert"] / base["us_per_insert"] - 1), 1)
|
||||
save("writes.json", results)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
CHECKSUM_SQL = {
|
||||
"users": "SELECT count(*) || ':' || md5(coalesce(string_agg(id::text || username || display_name || password_hash "
|
||||
"|| extract(epoch FROM created_at)::text, ',' ORDER BY id), '')) FROM users",
|
||||
"chats": "SELECT count(*) || ':' || md5(coalesce(string_agg(id::text || title || created_by "
|
||||
"|| extract(epoch FROM created_at)::text, ',' ORDER BY id), '')) FROM chats",
|
||||
"chat_members": "SELECT count(*) || ':' || md5(coalesce(string_agg(chat_id::text || user_id || position, ',' "
|
||||
"ORDER BY chat_id, user_id), '')) FROM chat_members",
|
||||
"messages_static": "SELECT count(*) || ':' || md5(coalesce(string_agg(id::text || chat_id || sender_id || md5(text) "
|
||||
"|| extract(epoch FROM created_at)::text, ',' ORDER BY id), '')) FROM messages "
|
||||
"WHERE text NOT LIKE 'marker-%'",
|
||||
"schema_migrations": "SELECT count(*) || ':' || md5(string_agg(version || checksum, ',' ORDER BY version)) "
|
||||
"FROM schema_migrations",
|
||||
"indexes": "SELECT count(*) || ':' || md5(string_agg(indexdef, ',' ORDER BY indexname)) FROM pg_indexes "
|
||||
"WHERE schemaname = 'public'",
|
||||
"constraints": "SELECT count(*) || ':' || md5(string_agg(conname || pg_get_constraintdef(oid), ',' ORDER BY conname)) "
|
||||
"FROM pg_constraint WHERE connamespace = 'public'::regnamespace",
|
||||
}
|
||||
|
||||
|
||||
def _writer(stop, counter):
|
||||
"""Пишет пронумерованные marker-сообщения по одному COMMIT, пока идёт backup."""
|
||||
member = one("SELECT chat_id || '|' || user_id FROM chat_members ORDER BY chat_id, position LIMIT 1")
|
||||
chat_id, user_id = member.split("|")
|
||||
proc = subprocess.Popen(shlex.split(SRC_PSQL) + ["-X", "-q", "-At", "-d", DB], stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, cwd=ROOT)
|
||||
n = 0
|
||||
try:
|
||||
while not stop.is_set():
|
||||
n += 1
|
||||
proc.stdin.write("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', 'marker-%06d'); "
|
||||
"SELECT %d;\n" % (chat_id, user_id, n, n))
|
||||
proc.stdin.flush()
|
||||
line = proc.stdout.readline() # ответ = COMMIT этой вставки подтверждён
|
||||
counter["acked"] = int(line.strip())
|
||||
counter.setdefault("t", []).append(time.monotonic())
|
||||
time.sleep(0.01)
|
||||
finally:
|
||||
proc.stdin.close()
|
||||
proc.wait(timeout=30)
|
||||
|
||||
|
||||
def step_backup(args):
|
||||
print("[backup] pg_dump -Fc → restore into a NEW server → reconcile")
|
||||
for t in ("messages",):
|
||||
psql("DELETE FROM %s WHERE text LIKE 'marker-%%';" % t)
|
||||
stop, counter = threading.Event(), {"acked": 0}
|
||||
writer = threading.Thread(target=_writer, args=(stop, counter))
|
||||
writer.start()
|
||||
time.sleep(1.0)
|
||||
before_dump = counter["acked"]
|
||||
t0 = time.monotonic()
|
||||
run(SRC_PG_DUMP, ["-Fc", "-Z", "6", "-d", DB, "-f", DUMP_PATH])
|
||||
dump_s = time.monotonic() - t0
|
||||
acked_at_dump_end = counter["acked"]
|
||||
time.sleep(1.0)
|
||||
stop.set()
|
||||
writer.join()
|
||||
total_markers = counter["acked"]
|
||||
dump_bytes = DUMP_HOST_PATH.stat().st_size if DUMP_HOST_PATH.exists() else None
|
||||
|
||||
# Новое хранилище: отдельный сервер/volume; база создаётся пустой, дамп накатывается в 4 потока.
|
||||
psql("DROP DATABASE IF EXISTS %s WITH (FORCE);" % RESTORED_DB, db="postgres", prefix=DST_PSQL)
|
||||
psql("CREATE DATABASE %s;" % RESTORED_DB, db="postgres", prefix=DST_PSQL)
|
||||
t1 = time.monotonic()
|
||||
run(DST_PG_RESTORE, ["-j", "4", "--no-owner", "--exit-on-error", "-d", RESTORED_DB, DUMP_PATH])
|
||||
restore_s = time.monotonic() - t1
|
||||
|
||||
t2 = time.monotonic()
|
||||
checks = {}
|
||||
for name, q in CHECKSUM_SQL.items():
|
||||
src = one(q)
|
||||
dst = one(q, db=RESTORED_DB, prefix=DST_PSQL)
|
||||
checks[name] = {"source": src.split(":")[0], "restored": dst.split(":")[0], "match": src == dst}
|
||||
# Выборочные сообщения: первое/последнее в 10 крупнейших чатах + 180 псевдослучайных (md5(id)).
|
||||
sample_sql = """
|
||||
WITH big AS (SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC, chat_id LIMIT 10),
|
||||
ends AS (SELECT (SELECT id FROM messages m WHERE m.chat_id = b.chat_id AND text NOT LIKE 'marker-%'
|
||||
ORDER BY created_at, id LIMIT 1) id FROM big b
|
||||
UNION SELECT (SELECT id FROM messages m WHERE m.chat_id = b.chat_id AND text NOT LIKE 'marker-%'
|
||||
ORDER BY created_at DESC, id DESC LIMIT 1) FROM big b),
|
||||
rnd AS (SELECT id FROM messages WHERE text NOT LIKE 'marker-%' ORDER BY md5(id::text) LIMIT 180)
|
||||
SELECT string_agg(id::text, ',' ORDER BY id) FROM (SELECT id FROM ends UNION SELECT id FROM rnd) x"""
|
||||
ids = one(sample_sql).split(",")
|
||||
fetch = ("SELECT json_agg(row_to_json(m) ORDER BY id)::text FROM (SELECT id, chat_id, sender_id, text, "
|
||||
"extract(epoch FROM created_at) AS ts FROM messages WHERE id = ANY('{%s}'::uuid[])) m" % ",".join(ids))
|
||||
sample_src, sample_dst = json.loads(one(fetch)), json.loads(one(fetch, db=RESTORED_DB, prefix=DST_PSQL))
|
||||
hot = one("SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC, chat_id LIMIT 1")
|
||||
page_q = ("SELECT md5(string_agg(id::text || md5(text), ',' ORDER BY created_at, id)) FROM (SELECT * FROM messages "
|
||||
"WHERE chat_id = '%s' AND text NOT LIKE 'marker-%%' ORDER BY created_at, id OFFSET 1000 LIMIT 50) p" % hot)
|
||||
markers_src = [int(x) for x in psql("SELECT substr(text, 8)::int FROM messages WHERE text LIKE 'marker-%' "
|
||||
"ORDER BY 1").split()]
|
||||
markers_dst = [int(x) for x in psql("SELECT substr(text, 8)::int FROM messages WHERE text LIKE 'marker-%' "
|
||||
"ORDER BY 1", db=RESTORED_DB, prefix=DST_PSQL).split()]
|
||||
verify_s = time.monotonic() - t2
|
||||
k = len(markers_dst)
|
||||
result = {
|
||||
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"dump": {"format": "custom (-Fc -Z 6)", "seconds": round(dump_s, 3), "bytes": dump_bytes},
|
||||
"restore": {"parallel_jobs": 4, "seconds": round(restore_s, 3)},
|
||||
"verify_seconds": round(verify_s, 3),
|
||||
"rto_dump_plus_restore_plus_verify_seconds": round(dump_s + restore_s + verify_s, 3),
|
||||
"table_checks": checks,
|
||||
"sample_messages": {"count": len(ids), "identical": sample_src == sample_dst},
|
||||
"history_page_hot_chat_offset_1000_identical": one(page_q) == one(page_q, db=RESTORED_DB, prefix=DST_PSQL),
|
||||
"concurrent_writes": {
|
||||
"acked_before_dump_started": before_dump,
|
||||
"acked_when_dump_finished": acked_at_dump_end,
|
||||
"acked_total": total_markers,
|
||||
"present_in_source": len(markers_src),
|
||||
"present_in_restored": k,
|
||||
"restored_is_contiguous_prefix": markers_dst == list(range(1, k + 1)),
|
||||
"lost_relative_to_source": len(markers_src) - k,
|
||||
},
|
||||
}
|
||||
result["all_match"] = (all(v["match"] for v in checks.values()) and result["sample_messages"]["identical"]
|
||||
and result["history_page_hot_chat_offset_1000_identical"]
|
||||
and result["concurrent_writes"]["restored_is_contiguous_prefix"])
|
||||
save("backup_restore.json", result)
|
||||
print(" dump %.2fs (%s bytes), restore %.2fs, verify %.2fs, all_match=%s, markers src=%d restored=%d"
|
||||
% (dump_s, dump_bytes, restore_s, verify_s, result["all_match"], len(markers_src), k))
|
||||
if not result["all_match"]:
|
||||
raise SystemExit("reconciliation FAILED")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("steps", nargs="*", default=["setup", "generate", "plans", "writes", "backup"])
|
||||
p.add_argument("--seed", type=float, default=0.42)
|
||||
p.add_argument("--users", type=int, default=2000)
|
||||
p.add_argument("--chats", type=int, default=300)
|
||||
p.add_argument("--messages", type=int, default=120000)
|
||||
p.add_argument("--zipf-s", type=float, default=1.1)
|
||||
p.add_argument("--repeats", type=int, default=15)
|
||||
p.add_argument("--write-rows", type=int, default=50000)
|
||||
p.add_argument("--write-runs", type=int, default=5)
|
||||
args = p.parse_args()
|
||||
steps = {"setup": step_setup, "generate": step_generate, "plans": step_plans, "writes": step_writes,
|
||||
"backup": step_backup}
|
||||
for name in args.steps:
|
||||
if name not in steps:
|
||||
raise SystemExit("unknown step " + name)
|
||||
steps[name](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
# Windows PowerShell 5.1+ equivalent of `make up && ./scripts/collect-evidence.sh`.
|
||||
# Starts the Compose stand, runs every check and writes outputs to evidence\ (no tokens/passwords).
|
||||
# Run from the repo root:
|
||||
# powershell -NoProfile -ExecutionPolicy Bypass -File scripts\collect-evidence.ps1
|
||||
# Optional: -SkipResearch (skip track 5.2), -SkipFailures (do not stop redis/postgres).
|
||||
param([switch]$SkipResearch, [switch]$SkipFailures)
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location $root
|
||||
$env:PYTHONUTF8 = '1'
|
||||
$env:PYTHONIOENCODING = 'utf-8'
|
||||
$ev = Join-Path $root 'evidence'
|
||||
New-Item -ItemType Directory -Force -Path $ev | Out-Null
|
||||
|
||||
if (Get-Command py -ErrorAction SilentlyContinue) { $script:PyExe = 'py'; $script:PyPre = @('-3') }
|
||||
elseif (Get-Command python -ErrorAction SilentlyContinue) { $script:PyExe = 'python'; $script:PyPre = @() }
|
||||
else { throw 'Python 3.9+ not found (install from python.org, enable "Add to PATH")' }
|
||||
function Py { & $script:PyExe @script:PyPre @args }
|
||||
function Dc { & docker compose --env-file .env @args }
|
||||
|
||||
function Get-Stamp {
|
||||
$commit = 'n/a'
|
||||
if (Get-Command git -ErrorAction SilentlyContinue) {
|
||||
$c = (& git rev-parse --short HEAD 2>$null)
|
||||
if ($LASTEXITCODE -eq 0 -and $c) { $commit = $c }
|
||||
}
|
||||
'# ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' commit=' + $commit + ' host=' + [Environment]::OSVersion.VersionString + ' cpus=' + [Environment]::ProcessorCount
|
||||
}
|
||||
function Invoke-Step([string]$name, [string]$title, [scriptblock]$block) {
|
||||
Write-Host ('>>> ' + $name) -ForegroundColor Cyan
|
||||
$global:LASTEXITCODE = 0
|
||||
$lines = @(& $block 2>&1 | ForEach-Object { "$_" })
|
||||
$code = $global:LASTEXITCODE
|
||||
$content = @((Get-Stamp), ('$ ' + $title)) + $lines + @('exit=' + $code)
|
||||
Set-Content -Path (Join-Path $ev ($name + '.txt')) -Encoding UTF8 -Value $content
|
||||
$lines | Select-Object -Last 2 | ForEach-Object { Write-Host (' ' + $_) }
|
||||
if ($code -ne 0) { Write-Host (' exit=' + $code + ' (see evidence\' + $name + '.txt)') -ForegroundColor Yellow }
|
||||
}
|
||||
|
||||
& docker version --format '{{.Server.Version}}' *> $null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Docker is not running: start Docker Desktop first' }
|
||||
|
||||
& (Join-Path $PSScriptRoot 'init-secrets.ps1')
|
||||
Set-Location $root
|
||||
Write-Host '>>> docker compose up (build, wait for healthchecks)' -ForegroundColor Cyan
|
||||
Dc --profile ttl up -d --build --wait
|
||||
if ($LASTEXITCODE -ne 0) { Dc logs --tail=80; throw 'docker compose up failed' }
|
||||
|
||||
Invoke-Step 'stack' 'docker version; docker compose version; images; ps' {
|
||||
& docker version --format 'docker {{.Server.Version}}'; Dc version; Dc --profile ttl images; Dc --profile ttl ps
|
||||
}
|
||||
Invoke-Step 'make-check' 'python scripts/check_contract.py' { Py scripts/check_contract.py }
|
||||
Invoke-Step 'smoke' 'python tests/smoke.py --base-url http://localhost:8080' { Py tests/smoke.py --base-url http://localhost:8080 }
|
||||
Invoke-Step 'test-api' 'python student-tests/test_api.py' { Py student-tests/test_api.py }
|
||||
Invoke-Step 'test-expiry' 'python student-tests/test_expiry.py # api-ttl, SESSION_TTL_SECONDS=2' { Py student-tests/test_expiry.py }
|
||||
Invoke-Step 'test-cache-race' 'docker compose run --rm --no-deps api: python student-tests/test_cache_race.py' {
|
||||
Dc run --rm --no-deps -T -v ((Join-Path $root 'student-tests') + ':/app/student-tests:ro') --entrypoint python api student-tests/test_cache_race.py
|
||||
}
|
||||
Set-Content -Path (Join-Path $ev 'restart-hook.txt') -Encoding UTF8 -Value @()
|
||||
# Windows: .cmd wrapper around restart.ps1; pwsh on Linux/macOS: the POSIX hook.
|
||||
$hook = if ($IsLinux -or $IsMacOS) { 'scripts/restart.sh' } else { 'scripts\restart.cmd' }
|
||||
Invoke-Step 'persistence' ('python tests/resilience.py --mode persistence --action-script ' + $hook) {
|
||||
Py tests/resilience.py --mode persistence --action-script $hook --base-url http://localhost:8080
|
||||
}
|
||||
if (-not $SkipFailures) {
|
||||
Invoke-Step 'test-failures' 'python student-tests/test_failures.py # stops/starts redis and postgres' { Py student-tests/test_failures.py }
|
||||
}
|
||||
Invoke-Step 'migrations-rerun' 'docker compose exec -T api python -m messenger migrate' { Dc exec -T api python -m messenger migrate }
|
||||
Invoke-Step 'load' 'python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json' {
|
||||
Py tests/load.py --base-url http://localhost:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||
}
|
||||
$race = @(Dc logs --no-color api 2>$null | Select-String -SimpleMatch 'registration race resolved').Count
|
||||
Set-Content -Path (Join-Path $ev 'race-unique-violations.txt') -Encoding ASCII -Value $race
|
||||
if (-not $SkipResearch) {
|
||||
Dc --profile research up -d --wait postgres-restore
|
||||
Invoke-Step 'track2' 'python research/track2.py' { Py research/track2.py }
|
||||
}
|
||||
Write-Host 'Done: evidence collected in evidence\' -ForegroundColor Green
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# Полный прогон доказательств на Compose-стенде; вывод — в evidence/ (без токенов и паролей).
|
||||
# Предусловие: make up (стенд запущен). Тест отказов и restart-hook перезапускают контейнеры.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
dc() { docker compose --env-file .env "$@"; }
|
||||
mkdir -p evidence
|
||||
stamp() { echo "# $(date -u +%FT%TZ) commit=$(git rev-parse --short HEAD 2>/dev/null || echo n/a) host=$(uname -srm)"; }
|
||||
run() { name=$1; shift; { stamp; echo "\$ $*"; "$@" 2>&1; echo "exit=$?"; } > "evidence/$name.txt" || true; tail -3 "evidence/$name.txt"; }
|
||||
|
||||
dc --profile ttl up -d --build --wait
|
||||
{ stamp; docker version --format 'docker {{.Server.Version}}'; dc version; dc --profile ttl images; dc --profile ttl ps; } > evidence/stack.txt 2>&1
|
||||
run make-check make check
|
||||
run smoke make test
|
||||
run test-api python3 student-tests/test_api.py
|
||||
run test-expiry python3 student-tests/test_expiry.py
|
||||
run test-cache-race make test-cache-race
|
||||
: > evidence/restart-hook.txt
|
||||
run persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
run test-failures python3 student-tests/test_failures.py
|
||||
run migrations-rerun dc exec -T api python -m messenger migrate
|
||||
run load python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
|
||||
dc logs --no-color api 2>/dev/null | grep -c 'registration race resolved' > evidence/race-unique-violations.txt || true
|
||||
dc --profile research up -d --wait postgres-restore
|
||||
run track2 python3 research/track2.py
|
||||
echo "evidence collected in evidence/"
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# Сбор доказательств на НАТИВНОМ стенде (PostgreSQL 16 + Redis 7 + API как процессы ОС).
|
||||
# Использовался в среде без доступа к Docker Hub; для сдачи предпочтителен scripts/collect-evidence.sh.
|
||||
# Ожидает: кластеры /srv/stand/pg (5432) и /srv/stand/pg-restore (5433), Redis 6379, API 8080/8081.
|
||||
set -u
|
||||
cd "$(dirname "$0")/../.."
|
||||
N=scripts/dev/native-ctl.sh
|
||||
export PGPASSWORD=${PGPASSWORD:-$(cat /srv/stand/secrets/pg_password)}
|
||||
export PSQL="psql -h 127.0.0.1 -U messenger -d messenger"
|
||||
export STOP_DB="$N stop db" START_DB="$N start db" STOP_CACHE="$N stop cache" START_CACHE="$N start cache"
|
||||
export SRC_PSQL="psql -h 127.0.0.1 -p 5432 -U messenger" SRC_PG_DUMP="pg_dump -h 127.0.0.1 -p 5432 -U messenger"
|
||||
export DST_PSQL="psql -h 127.0.0.1 -p 5433 -U messenger" DST_PG_RESTORE="pg_restore -h 127.0.0.1 -p 5433 -U messenger"
|
||||
export DUMP_PATH=/srv/stand/backups/research.dump DUMP_HOST_PATH=/srv/stand/backups/research.dump
|
||||
stamp() { echo "# $(date -u +%FT%TZ) native stand; $(uname -srm); nproc=$(nproc); mem=$(free -m | awk '/Mem/{print $2}')MiB"; }
|
||||
run() { name=$1; shift; { stamp; echo "\$ $*"; "$@" 2>&1; echo "exit=$?"; } > "evidence/$name.txt"; tail -2 "evidence/$name.txt"; }
|
||||
mkdir -p evidence
|
||||
run make-check make check
|
||||
run smoke make test
|
||||
run test-api python3 student-tests/test_api.py
|
||||
grep -c 'registration race resolved' /srv/stand/api.log > evidence/race-unique-violations.txt
|
||||
run test-expiry python3 student-tests/test_expiry.py
|
||||
: > /srv/stand/restart-evidence.log
|
||||
run persistence make test-persistence ACTION_SCRIPT=scripts/dev/restart-native.sh
|
||||
cp /srv/stand/restart-evidence.log evidence/restart-hook.txt
|
||||
run test-failures python3 student-tests/test_failures.py
|
||||
run load python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# Управление нативным стендом (без Docker): native-ctl.sh stop|start db|cache|api|api-ttl
|
||||
set -eu
|
||||
STAND=${STAND:-/srv/stand}
|
||||
RPW=$(cat "$STAND/secrets/redis_password") # синтетический пароль стенда, вне репозитория
|
||||
case "$1:$2" in
|
||||
stop:db) su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $STAND/pg -m fast -w stop" >/dev/null ;;
|
||||
start:db) su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $STAND/pg -w start -l $STAND/pg.log -o '-p 5432 -k /run/postgresql'" >/dev/null ;;
|
||||
stop:cache) redis-cli -a "$RPW" --no-auth-warning shutdown nosave >/dev/null 2>&1 || true ;;
|
||||
start:cache) redis-server --port 6379 --save '' --appendonly no --requirepass "$RPW" --maxmemory 64mb \
|
||||
--maxmemory-policy allkeys-lru --daemonize yes --dir "$STAND/redis" --logfile "$STAND/redis.log" ;;
|
||||
stop:api|stop:api-ttl) kill "$(cat "$STAND/$2.pid")"; while kill -0 "$(cat "$STAND/$2.pid")" 2>/dev/null; do sleep 0.2; done ;;
|
||||
start:api) "$STAND/start-api.sh" api 8080 3600 ;;
|
||||
start:api-ttl) "$STAND/start-api.sh" api-ttl 8081 2 ;;
|
||||
*) echo "usage: $0 stop|start db|cache|api|api-ttl" >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Хук перезапуска для НАТИВНОГО стенда без Docker (как собирались доказательства в среде без
|
||||
# доступа к Docker Hub). Основной хук для сдачи — scripts/restart.sh (Compose).
|
||||
# Перезапускает API, PostgreSQL и Redis, сохраняя каталоги данных. Ничего не удаляет и не seed-ит.
|
||||
set -eu
|
||||
STAND=${STAND:-/srv/stand}
|
||||
RPW=$(cat "$STAND/secrets/redis_password") # синтетический пароль стенда, вне репозитория
|
||||
LOG=${RESTART_EVIDENCE:-$STAND/restart-evidence.log}
|
||||
pids() {
|
||||
printf 'api=%s postgres=%s redis=%s\n' \
|
||||
"$(cat "$STAND/api.pid" 2>/dev/null || echo -)" \
|
||||
"$(head -1 "$STAND/pg/postmaster.pid" 2>/dev/null || echo -)" \
|
||||
"$(redis-cli -a "$RPW" --no-auth-warning info server 2>/dev/null | sed -n 's/^process_id:\(.*\)\r$/\1/p')"
|
||||
}
|
||||
{ printf '%s before: ' "$(date -u +%FT%TZ)"; pids; } >> "$LOG"
|
||||
kill "$(cat "$STAND/api.pid")" 2>/dev/null || true
|
||||
while kill -0 "$(cat "$STAND/api.pid")" 2>/dev/null; do sleep 0.2; done
|
||||
su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D $STAND/pg -m fast -w restart -l $STAND/pg.log -o '-p 5432 -k /run/postgresql'" >/dev/null
|
||||
redis-cli -a "$RPW" --no-auth-warning shutdown nosave >/dev/null 2>&1 || true
|
||||
sleep 0.3
|
||||
redis-server --port 6379 --save '' --appendonly no --requirepass "$RPW" --maxmemory 64mb \
|
||||
--maxmemory-policy allkeys-lru --daemonize yes --dir "$STAND/redis" --logfile "$STAND/redis.log"
|
||||
"$STAND/start-api.sh" api 8080 3600
|
||||
sleep 0.5
|
||||
{ printf '%s after: ' "$(date -u +%FT%TZ)"; pids; } >> "$LOG"
|
||||
@@ -0,0 +1,36 @@
|
||||
# Windows PowerShell 5.1+ version of scripts/init-secrets.sh.
|
||||
# Creates local synthetic secrets in secrets\ and .env (both git-ignored).
|
||||
# Idempotent: existing files are NOT overwritten. Values are never printed.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location $root
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $root 'secrets'), (Join-Path $root 'backups') | Out-Null
|
||||
$ascii = New-Object System.Text.ASCIIEncoding
|
||||
|
||||
function New-Hex {
|
||||
$bytes = New-Object byte[] 24
|
||||
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
$rng.GetBytes($bytes)
|
||||
($bytes | ForEach-Object { $_.ToString('x2') }) -join ''
|
||||
}
|
||||
function Set-SecretFile([string]$name, [string]$value) {
|
||||
$path = Join-Path (Join-Path $root 'secrets') $name
|
||||
if (-not (Test-Path $path)) {
|
||||
# No BOM, no trailing CR/LF: the value is read verbatim inside Linux containers.
|
||||
[System.IO.File]::WriteAllText($path, $value, $ascii)
|
||||
}
|
||||
}
|
||||
function Get-SecretFile([string]$name) {
|
||||
([System.IO.File]::ReadAllText((Join-Path (Join-Path $root 'secrets') $name))).Trim()
|
||||
}
|
||||
|
||||
Set-SecretFile 'postgres_password' (New-Hex)
|
||||
Set-SecretFile 'redis_password' (New-Hex)
|
||||
$pg = Get-SecretFile 'postgres_password'
|
||||
$rd = Get-SecretFile 'redis_password'
|
||||
Set-SecretFile 'database_url' ("postgresql://messenger:" + $pg + "@postgres:5432/messenger")
|
||||
Set-SecretFile 'cache_url' ("redis://:" + $rd + "@redis:6379/0")
|
||||
if (-not (Test-Path (Join-Path $root '.env'))) {
|
||||
Copy-Item (Join-Path $root '.env.example') (Join-Path $root '.env')
|
||||
}
|
||||
Write-Output 'secrets\ and .env are ready (values not shown).'
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные синтетические secrets для стенда (каталог secrets/ в .gitignore).
|
||||
# Идемпотентно: существующие файлы НЕ перезаписываются (иначе пароль разойдётся с уже
|
||||
# инициализированным volume PostgreSQL). Значения не печатаются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets backups
|
||||
chmod 700 secrets
|
||||
gen() { python3 -c 'import secrets; print(secrets.token_hex(24))'; }
|
||||
if [ ! -f secrets/postgres_password ]; then gen > secrets/postgres_password; fi
|
||||
if [ ! -f secrets/redis_password ]; then gen > secrets/redis_password; fi
|
||||
PG=$(cat secrets/postgres_password)
|
||||
RD=$(cat secrets/redis_password)
|
||||
[ -f secrets/database_url ] || printf 'postgresql://messenger:%s@postgres:5432/messenger\n' "$PG" > secrets/database_url
|
||||
[ -f secrets/cache_url ] || printf 'redis://:%s@redis:6379/0\n' "$RD" > secrets/cache_url
|
||||
# Файлы монтируются в контейнеры как /run/secrets/*; процесс API работает под uid 10001,
|
||||
# поэтому файл должен быть читаем внутри контейнера. Каталог secrets/ закрыт (700) на хосте.
|
||||
chmod 644 secrets/*
|
||||
[ -f .env ] || cp .env.example .env
|
||||
echo "secrets/ and .env are ready (values not shown)."
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
rem Windows wrapper: tests\resilience.py starts the hook directly; .cmd files run without a shell.
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0restart.ps1"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,34 @@
|
||||
# Windows PowerShell version of scripts/restart.sh (restart hook for the persistence test).
|
||||
# Restarts API and ALL used stores (PostgreSQL, Redis) keeping volumes. No down -v, no flush, no re-seed.
|
||||
# Container state before/after (StartedAt, PID) is appended to evidence\restart-hook.txt.
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location $root
|
||||
$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' }
|
||||
$log = if ($env:RESTART_EVIDENCE) { $env:RESTART_EVIDENCE } else { Join-Path $root 'evidence\restart-hook.txt' }
|
||||
|
||||
function Invoke-Compose {
|
||||
& docker compose --env-file $envFile @args
|
||||
if ($LASTEXITCODE -ne 0) { throw ('docker compose ' + ($args -join ' ') + ' failed') }
|
||||
}
|
||||
function Get-State {
|
||||
foreach ($svc in 'api', 'postgres', 'redis') {
|
||||
$cid = (& docker compose --env-file $envFile ps -q $svc | Select-Object -First 1)
|
||||
& docker inspect -f ($svc + ' status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}') $cid
|
||||
}
|
||||
& docker volume ls --format '{{.Name}}' | Where-Object { $_ -match 'pgdata$' } | ForEach-Object { 'volume kept: ' + $_ }
|
||||
}
|
||||
|
||||
try {
|
||||
Add-Content -Path $log -Encoding UTF8 -Value ('== ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' before restart')
|
||||
Add-Content -Path $log -Encoding UTF8 -Value (Get-State)
|
||||
Invoke-Compose stop api
|
||||
Invoke-Compose restart postgres redis
|
||||
Invoke-Compose start api
|
||||
Add-Content -Path $log -Encoding UTF8 -Value ('== ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' after restart')
|
||||
Add-Content -Path $log -Encoding UTF8 -Value (Get-State)
|
||||
exit 0
|
||||
} catch {
|
||||
[Console]::Error.WriteLine($_.Exception.Message)
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Restart hook для make test-persistence (Compose).
|
||||
# Перезапускает API и ВСЕ используемые хранилища (PostgreSQL, Redis), сохраняя volumes.
|
||||
# Никаких down -v / FLUSH / повторного seed. Состояние контейнеров до/после (StartedAt, PID)
|
||||
# дописывается в evidence/restart-hook.txt — доказательство, что процессы действительно сменились.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE=${ENV_FILE:-.env}
|
||||
dc() { docker compose --env-file "$ENV_FILE" "$@"; }
|
||||
LOG=${RESTART_EVIDENCE:-evidence/restart-hook.txt}
|
||||
state() {
|
||||
for svc in api postgres redis; do
|
||||
cid=$(dc ps -q "$svc")
|
||||
docker inspect -f "$svc status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}" "$cid"
|
||||
done
|
||||
docker volume ls --format '{{.Name}}' | grep -E 'pgdata$' | sed 's/^/volume kept: /'
|
||||
}
|
||||
{ echo "== $(date -u +%FT%TZ) before restart"; state; } >> "$LOG"
|
||||
dc stop api
|
||||
dc restart postgres redis
|
||||
dc start api
|
||||
{ echo "== $(date -u +%FT%TZ) after restart"; state; } >> "$LOG"
|
||||
@@ -0,0 +1,2 @@
|
||||
"""MTUSI messenger API, lab2."""
|
||||
__version__ = "2.0.0"
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Точка входа: python -m messenger [serve|migrate|healthcheck]."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
_STD = set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message", "asctime"}
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Структурированный лог в stdout. Токены, пароли, Authorization и тела запросов не логируются."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
data = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + "Z",
|
||||
"level": record.levelname.lower(), "logger": record.name, "msg": record.getMessage()}
|
||||
data.update({k: v for k, v in vars(record).items() if k not in _STD and not k.startswith("_")})
|
||||
if record.exc_info:
|
||||
data["exc"] = self.formatException(record.exc_info).splitlines()[-1]
|
||||
return json.dumps(data, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), handlers=[handler], force=True)
|
||||
logging.getLogger("uvicorn.access").disabled = True # свой access-log без query-строк
|
||||
|
||||
|
||||
def serve() -> int:
|
||||
import uvicorn
|
||||
|
||||
from .app import build_app
|
||||
from .config import load
|
||||
|
||||
settings = load()
|
||||
app = build_app(settings)
|
||||
config = uvicorn.Config(app, host=settings.host, port=settings.port, log_config=None,
|
||||
access_log=False, server_header=False, proxy_headers=False,
|
||||
timeout_keep_alive=5, timeout_graceful_shutdown=10, lifespan="on")
|
||||
server = uvicorn.Server(config)
|
||||
server.run()
|
||||
return 0 if server.started else 1
|
||||
|
||||
|
||||
def migrate() -> int:
|
||||
from .config import load
|
||||
from .db import migrate_with_wait
|
||||
|
||||
applied = asyncio.run(migrate_with_wait(load()))
|
||||
print(json.dumps({"applied": applied}))
|
||||
return 0
|
||||
|
||||
|
||||
def healthcheck() -> int:
|
||||
port = os.environ.get("APP_PORT", "8080")
|
||||
path = sys.argv[2] if len(sys.argv) > 2 else "/health/ready"
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=2) as resp:
|
||||
return 0 if resp.status == 200 else 1
|
||||
except Exception: # noqa: BLE001
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
setup_logging()
|
||||
command = sys.argv[1] if len(sys.argv) > 1 else "serve"
|
||||
commands = {"serve": serve, "migrate": migrate, "healthcheck": healthcheck}
|
||||
if command not in commands:
|
||||
print("usage: python -m messenger [serve|migrate|healthcheck]", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
return commands[command]()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
from .config import ConfigError
|
||||
if isinstance(exc, ConfigError):
|
||||
logging.getLogger("messenger").error("configuration error: %s", exc)
|
||||
return 2
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,510 @@
|
||||
"""HTTP API мессенджера (lab2): Starlette + psycopg 3 (async pool) + Redis/Valkey-кэш сессий."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import contextlib
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from psycopg import errors as pg_errors
|
||||
from starlette.applications import Starlette
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, Response
|
||||
from starlette.routing import Route
|
||||
|
||||
from . import db
|
||||
from .config import Settings
|
||||
from .errors import ApiError, invalid, not_found, unauthorized_basic, unauthorized_bearer, unavailable
|
||||
from .security import PasswordHasher, looks_like_token, new_token, token_hash
|
||||
from .sessions import Principal, SessionCache
|
||||
from .validation import (USERNAME_RE, decode_cursor, encode_cursor, parse_json_object, parse_limit,
|
||||
parse_uuid, single_param, text_field)
|
||||
|
||||
log = logging.getLogger("messenger.http")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Ответы и обвязка
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
def request_id(request: Request) -> str:
|
||||
rid = getattr(request.state, "request_id", None)
|
||||
if not rid:
|
||||
rid = uuid.uuid4().hex
|
||||
request.state.request_id = rid
|
||||
return rid
|
||||
|
||||
|
||||
def json_response(request: Request, payload, status: int = 200, headers: dict | None = None) -> JSONResponse:
|
||||
h = {"X-Request-Id": request_id(request), "Cache-Control": "no-store"}
|
||||
h.update(headers or {})
|
||||
return JSONResponse(payload, status_code=status, headers=h)
|
||||
|
||||
|
||||
def error_response(request: Request, err: ApiError) -> JSONResponse:
|
||||
rid = request_id(request)
|
||||
body = {"error": {"code": err.public_code, "message": err.message, "request_id": rid}}
|
||||
return json_response(request, body, err.status, err.headers)
|
||||
|
||||
|
||||
class RequestContextMiddleware:
|
||||
"""Чистый ASGI middleware: X-Request-Id в каждом ответе (включая 404/405 роутера) и access-log
|
||||
без query-строки, заголовков и тела."""
|
||||
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
rid = uuid.uuid4().hex
|
||||
scope.setdefault("state", {})["request_id"] = rid
|
||||
started = time.monotonic()
|
||||
status_holder = {}
|
||||
|
||||
async def send_wrapper(message):
|
||||
if message["type"] == "http.response.start":
|
||||
status_holder["status"] = message["status"]
|
||||
headers = list(message.get("headers", []))
|
||||
if not any(k.lower() == b"x-request-id" for k, _ in headers):
|
||||
headers.append((b"x-request-id", rid.encode()))
|
||||
message["headers"] = headers
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
finally:
|
||||
log.info("request", extra={"request_id": rid, "method": scope.get("method"),
|
||||
"path": scope.get("path"), "status": status_holder.get("status"),
|
||||
"duration_ms": round((time.monotonic() - started) * 1000, 2)})
|
||||
|
||||
|
||||
def endpoint(handler):
|
||||
"""Единая обработка ошибок и общий таймаут запроса."""
|
||||
async def wrapper(request: Request):
|
||||
state: AppState = request.app.state.messenger
|
||||
try:
|
||||
async with asyncio.timeout(state.settings.request_timeout_seconds):
|
||||
return await handler(request, state)
|
||||
except ApiError as err:
|
||||
return error_response(request, err)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if db.is_unavailable(exc) or isinstance(exc, TimeoutError):
|
||||
log.warning("dependency unavailable", extra={"request_id": request_id(request),
|
||||
"error": type(exc).__name__})
|
||||
return error_response(request, unavailable())
|
||||
log.exception("unhandled error", extra={"request_id": request_id(request)})
|
||||
return error_response(request, ApiError("internal_error"))
|
||||
wrapper.__name__ = handler.__name__
|
||||
return wrapper
|
||||
|
||||
|
||||
async def read_body(request: Request, limit: int) -> bytes:
|
||||
declared = request.headers.get("content-length")
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise invalid("Некорректный Content-Length")
|
||||
if int(declared) > limit:
|
||||
raise ApiError("payload_too_large")
|
||||
chunks, size = [], 0
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
raise ApiError("payload_too_large")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
async def read_json(request: Request, state: "AppState", allowed: set[str], required: set[str]) -> dict:
|
||||
ctype = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
body = await read_body(request, state.settings.max_body_bytes)
|
||||
if ctype != "application/json":
|
||||
raise invalid("Ожидается Content-Type: application/json")
|
||||
return parse_json_object(body, allowed, required)
|
||||
|
||||
|
||||
def user_json(row) -> dict:
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
|
||||
"created_at": db.iso(row["created_at"])}
|
||||
|
||||
|
||||
def chat_json(row) -> dict:
|
||||
return {"id": str(row["id"]), "title": row["title"],
|
||||
"member_ids": [str(m) for m in row["member_ids"]], "created_at": db.iso(row["created_at"])}
|
||||
|
||||
|
||||
def message_json(row) -> dict:
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
|
||||
"text": row["text"], "created_at": db.iso(row["created_at"])}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Состояние приложения и аутентификация
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
class AppState:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.pool = db.make_pool(settings)
|
||||
self.cache = SessionCache(settings)
|
||||
self.hasher = PasswordHasher(settings.scrypt_log2_n, settings.scrypt_r, settings.scrypt_p,
|
||||
settings.kdf_concurrency)
|
||||
self.ready = False
|
||||
self.stop = asyncio.Event()
|
||||
self.tasks: list[asyncio.Task] = []
|
||||
|
||||
|
||||
async def authenticate(request: Request, state: AppState) -> Principal:
|
||||
"""Только Authorization: Bearer <opaque>. X-User-Id и Basic игнорируются/отклоняются."""
|
||||
header = request.headers.get("authorization", "")
|
||||
scheme, _, token = header.partition(" ")
|
||||
token = token.strip()
|
||||
if scheme.lower() != "bearer" or not looks_like_token(token):
|
||||
raise unauthorized_bearer()
|
||||
th = token_hash(token)
|
||||
principal = await state.cache.get(th)
|
||||
if principal is not None:
|
||||
return principal
|
||||
async with state.pool.connection() as conn:
|
||||
row = await (await conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
|
||||
" FROM sessions s JOIN users u ON u.id = s.user_id"
|
||||
" WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp()",
|
||||
(th,))).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized_bearer()
|
||||
principal = Principal(user=user_json(row), expires_at=row["expires_at"].timestamp(), token_hash=th)
|
||||
await state.cache.put(principal)
|
||||
return principal
|
||||
|
||||
|
||||
def parse_basic(header: str) -> tuple[str, str]:
|
||||
scheme, _, value = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized_basic()
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized_basic() from None
|
||||
username, sep, password = decoded.partition(":") # двоеточие допустимо только в пароле
|
||||
if not sep or not username.isascii() or not USERNAME_RE.fullmatch(username) or not 12 <= len(password) <= 128:
|
||||
raise unauthorized_basic()
|
||||
return username, password
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Handlers
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
@endpoint
|
||||
async def live(request: Request, state: AppState):
|
||||
return json_response(request, {"status": "ok"})
|
||||
|
||||
|
||||
@endpoint
|
||||
async def ready(request: Request, state: AppState):
|
||||
# Критичная зависимость синхронного пути — только PostgreSQL. Redis — кэш: при его потере
|
||||
# API продолжает работу через источник истины, поэтому readiness от него не зависит.
|
||||
if not state.ready:
|
||||
raise unavailable()
|
||||
try:
|
||||
async with asyncio.timeout(1.5):
|
||||
async with state.pool.connection() as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
except Exception: # noqa: BLE001
|
||||
raise unavailable() from None
|
||||
return json_response(request, {"status": "ok"})
|
||||
|
||||
|
||||
@endpoint
|
||||
async def create_user(request: Request, state: AppState):
|
||||
data = await read_json(request, state, {"username", "display_name", "password"},
|
||||
{"username", "display_name", "password"})
|
||||
username = data["username"]
|
||||
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
|
||||
raise invalid("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
display_name = text_field(data, "display_name", 1, 100)
|
||||
password = text_field(data, "password", 12, 128)
|
||||
async with state.pool.connection() as conn:
|
||||
taken = await (await conn.execute("SELECT 1 FROM users WHERE username = %s", (username,))).fetchone()
|
||||
if taken:
|
||||
raise ApiError("conflict", "username уже занят")
|
||||
password_hash = await state.hasher.hash(password)
|
||||
try:
|
||||
async with state.pool.connection() as conn:
|
||||
row = await (await conn.execute(
|
||||
"INSERT INTO users (username, display_name, password_hash) VALUES (%s, %s, %s)"
|
||||
" RETURNING id, username, display_name, created_at",
|
||||
(username, display_name, password_hash))).fetchone()
|
||||
except pg_errors.UniqueViolation:
|
||||
# Гонка регистрации: проверку выше прошли оба запроса, уникальный индекс пропустил один.
|
||||
log.info("registration race resolved by unique constraint", extra={"request_id": request_id(request)})
|
||||
raise ApiError("conflict", "username уже занят") from None
|
||||
return json_response(request, user_json(row), 201)
|
||||
|
||||
|
||||
@endpoint
|
||||
async def me(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
return json_response(request, principal.user)
|
||||
|
||||
|
||||
@endpoint
|
||||
async def create_session(request: Request, state: AppState):
|
||||
username, password = parse_basic(request.headers.get("authorization", ""))
|
||||
body = await read_body(request, 1024)
|
||||
if body:
|
||||
raise invalid("POST /auth/sessions не принимает тело")
|
||||
async with state.pool.connection() as conn:
|
||||
row = await (await conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,))).fetchone()
|
||||
ok = await state.hasher.verify(password, row["password_hash"] if row else None)
|
||||
if not ok:
|
||||
raise unauthorized_basic()
|
||||
token = new_token()
|
||||
th = token_hash(token)
|
||||
async with state.pool.connection() as conn:
|
||||
srow = await (await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at)"
|
||||
" VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
|
||||
(th, row["id"], state.settings.session_ttl_seconds))).fetchone()
|
||||
user = user_json(row)
|
||||
await state.cache.put(Principal(user=user, expires_at=srow["expires_at"].timestamp(), token_hash=th))
|
||||
return json_response(request, {"token": token, "token_type": "Bearer",
|
||||
"expires_at": db.iso(srow["expires_at"]), "user": user}, 201)
|
||||
|
||||
|
||||
@endpoint
|
||||
async def delete_session(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
th = principal.token_hash
|
||||
async with state.pool.connection() as conn:
|
||||
async with conn.transaction():
|
||||
revoked = await (await conn.execute(
|
||||
"UPDATE sessions SET revoked_at = clock_timestamp()"
|
||||
" WHERE token_hash = %s AND revoked_at IS NULL RETURNING 1", (th,))).fetchone()
|
||||
if revoked is None:
|
||||
raise unauthorized_bearer()
|
||||
if state.cache.enabled:
|
||||
await conn.execute(
|
||||
"INSERT INTO session_cache_invalidations (token_hash) VALUES (%s) ON CONFLICT DO NOTHING",
|
||||
(th,))
|
||||
# COMMIT в PostgreSQL уже сделал отзыв окончательным; Redis лишь догоняет источник истины.
|
||||
if state.cache.enabled and await state.cache.invalidate(th):
|
||||
with contextlib.suppress(Exception):
|
||||
async with state.pool.connection() as conn:
|
||||
await conn.execute("DELETE FROM session_cache_invalidations WHERE token_hash = %s", (th,))
|
||||
return Response(status_code=204, headers={"X-Request-Id": request_id(request)})
|
||||
|
||||
|
||||
@endpoint
|
||||
async def create_chat(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
data = await read_json(request, state, {"title", "member_ids"}, {"title", "member_ids"})
|
||||
title = text_field(data, "title", 1, 100)
|
||||
raw_members = data["member_ids"]
|
||||
if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100:
|
||||
raise invalid("member_ids: массив из 1–100 UUID")
|
||||
members = [parse_uuid(m, "member_id") for m in raw_members]
|
||||
if len(set(members)) != len(members):
|
||||
raise invalid("member_ids содержит дубликаты")
|
||||
caller = uuid.UUID(principal.user["id"])
|
||||
if caller not in members:
|
||||
raise invalid("member_ids должен содержать создателя чата")
|
||||
async with state.pool.connection() as conn:
|
||||
try:
|
||||
async with conn.transaction(): # атомарно: чат и весь состав или ничего
|
||||
chat = await (await conn.execute(
|
||||
"INSERT INTO chats (title, created_by) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||
(title, caller))).fetchone()
|
||||
await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position)"
|
||||
" SELECT %s, m.user_id, m.ord - 1 FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
|
||||
(chat["id"], members))
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise invalid("member_ids содержит неизвестного пользователя") from None
|
||||
chat["member_ids"] = members
|
||||
return json_response(request, chat_json(chat), 201)
|
||||
|
||||
|
||||
_CHAT_LIST_SQL = (
|
||||
"SELECT c.id, c.title, c.created_at,"
|
||||
" (SELECT array_agg(x.user_id ORDER BY x.position) FROM chat_members x WHERE x.chat_id = c.id) AS member_ids"
|
||||
" FROM chat_members m JOIN chats c ON c.id = m.chat_id"
|
||||
" WHERE m.user_id = %s {after}"
|
||||
" ORDER BY c.created_at, c.id LIMIT %s")
|
||||
|
||||
|
||||
@endpoint
|
||||
async def list_chats(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
caller = uuid.UUID(principal.user["id"])
|
||||
limit = parse_limit(request.query_params)
|
||||
cursor = single_param(request.query_params, "cursor")
|
||||
async with state.pool.connection() as conn:
|
||||
if cursor is not None:
|
||||
ts, last_id = decode_cursor(cursor, "chats", caller)
|
||||
rows = await (await conn.execute(_CHAT_LIST_SQL.format(after="AND (c.created_at, c.id) > (%s, %s)"),
|
||||
(caller, ts, last_id, limit + 1))).fetchall()
|
||||
else:
|
||||
rows = await (await conn.execute(_CHAT_LIST_SQL.format(after=""), (caller, limit + 1))).fetchall()
|
||||
items = rows[:limit]
|
||||
next_cursor = (encode_cursor("chats", caller, items[-1]["created_at"], items[-1]["id"])
|
||||
if len(rows) > limit else None)
|
||||
return json_response(request, {"items": [chat_json(r) for r in items], "next_cursor": next_cursor})
|
||||
|
||||
|
||||
@endpoint
|
||||
async def get_chat(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
chat_id = parse_uuid(request.path_params["chat_id"], "chat_id")
|
||||
async with state.pool.connection() as conn:
|
||||
row = await (await conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at, array_agg(m.user_id ORDER BY m.position) AS member_ids"
|
||||
" FROM chats c JOIN chat_members m ON m.chat_id = c.id"
|
||||
" WHERE c.id = %s AND EXISTS (SELECT 1 FROM chat_members a WHERE a.chat_id = c.id AND a.user_id = %s)"
|
||||
" GROUP BY c.id", (chat_id, uuid.UUID(principal.user["id"])))).fetchone()
|
||||
if row is None:
|
||||
raise not_found() # нет или чужой — одинаковый ответ
|
||||
return json_response(request, chat_json(row))
|
||||
|
||||
|
||||
async def _is_member(conn, chat_id: uuid.UUID, user_id: uuid.UUID) -> bool:
|
||||
row = await (await conn.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id))).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
@endpoint
|
||||
async def create_message(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
chat_id = parse_uuid(request.path_params["chat_id"], "chat_id")
|
||||
sender = uuid.UUID(principal.user["id"])
|
||||
try:
|
||||
data = await read_json(request, state, {"text"}, {"text"})
|
||||
text = text_field(data, "text", 1, 4000)
|
||||
except ApiError as err:
|
||||
if err.status == 400:
|
||||
# Посторонний не должен по 400/404 узнавать о существовании чата.
|
||||
async with state.pool.connection() as conn:
|
||||
if not await _is_member(conn, chat_id, sender):
|
||||
raise not_found() from None
|
||||
raise
|
||||
async with state.pool.connection() as conn:
|
||||
row = await (await conn.execute(
|
||||
"INSERT INTO messages (chat_id, sender_id, text)"
|
||||
" SELECT %(c)s, %(u)s, %(t)s"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %(c)s AND user_id = %(u)s)"
|
||||
" RETURNING id, chat_id, sender_id, text, created_at",
|
||||
{"c": chat_id, "u": sender, "t": text})).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return json_response(request, message_json(row), 201)
|
||||
|
||||
|
||||
_HISTORY_SQL = (
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages"
|
||||
" WHERE chat_id = %s {after} ORDER BY created_at, id LIMIT %s")
|
||||
|
||||
|
||||
@endpoint
|
||||
async def list_messages(request: Request, state: AppState):
|
||||
principal = await authenticate(request, state)
|
||||
chat_id = parse_uuid(request.path_params["chat_id"], "chat_id")
|
||||
async with state.pool.connection() as conn:
|
||||
if not await _is_member(conn, chat_id, uuid.UUID(principal.user["id"])):
|
||||
raise not_found()
|
||||
limit = parse_limit(request.query_params)
|
||||
cursor = single_param(request.query_params, "cursor")
|
||||
if cursor is not None:
|
||||
ts, last_id = decode_cursor(cursor, "messages", chat_id)
|
||||
rows = await (await conn.execute(_HISTORY_SQL.format(after="AND (created_at, id) > (%s, %s)"),
|
||||
(chat_id, ts, last_id, limit + 1))).fetchall()
|
||||
else:
|
||||
rows = await (await conn.execute(_HISTORY_SQL.format(after=""), (chat_id, limit + 1))).fetchall()
|
||||
items = rows[:limit]
|
||||
next_cursor = (encode_cursor("messages", chat_id, items[-1]["created_at"], items[-1]["id"])
|
||||
if len(rows) > limit else None)
|
||||
return json_response(request, {"items": [message_json(r) for r in items], "next_cursor": next_cursor})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Фоновые задачи и сборка приложения
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
async def expired_sessions_cleanup(state: AppState) -> None:
|
||||
"""Удаляет сессии, истёкшие более суток назад, пачками (не держит долгих блокировок)."""
|
||||
while not state.stop.is_set():
|
||||
with contextlib.suppress(Exception):
|
||||
async with state.pool.connection() as conn:
|
||||
await conn.execute(
|
||||
"DELETE FROM sessions WHERE token_hash IN (SELECT token_hash FROM sessions"
|
||||
" WHERE expires_at < clock_timestamp() - interval '1 day' LIMIT 1000)")
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(state.stop.wait(), timeout=600)
|
||||
|
||||
|
||||
def build_app(settings: Settings) -> Starlette:
|
||||
state = AppState(settings)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def lifespan(app: Starlette):
|
||||
if settings.migrate_on_start:
|
||||
applied = await db.migrate_with_wait(settings)
|
||||
log.info("migrations checked", extra={"applied": applied})
|
||||
await state.pool.open(wait=False)
|
||||
state.ready = True
|
||||
if state.cache.enabled:
|
||||
state.tasks.append(asyncio.create_task(state.cache.maintenance_loop(state.pool, state.stop)))
|
||||
state.tasks.append(asyncio.create_task(expired_sessions_cleanup(state)))
|
||||
log.info("api started", extra={"cache": state.cache.enabled,
|
||||
"session_ttl_seconds": settings.session_ttl_seconds})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
state.ready = False
|
||||
state.stop.set()
|
||||
for task in state.tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*state.tasks, return_exceptions=True)
|
||||
await state.cache.close()
|
||||
await state.pool.close()
|
||||
|
||||
async def http_exception(request: Request, exc: HTTPException):
|
||||
if exc.status_code == 405:
|
||||
err = ApiError("method_not_allowed", "Метод не поддерживается", headers=dict(exc.headers or {}))
|
||||
elif exc.status_code == 404:
|
||||
err = ApiError("not_found")
|
||||
else:
|
||||
err = ApiError("invalid_request")
|
||||
return error_response(request, err)
|
||||
|
||||
async def server_error(request: Request, exc: Exception):
|
||||
return error_response(request, ApiError("internal_error"))
|
||||
|
||||
routes = [
|
||||
Route("/health/live", live, methods=["GET"]),
|
||||
Route("/health/ready", ready, methods=["GET"]),
|
||||
Route("/api/v1/users", create_user, methods=["POST"]),
|
||||
Route("/api/v1/users/me", me, methods=["GET"]),
|
||||
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
|
||||
Route("/api/v1/auth/sessions/current", delete_session, methods=["DELETE"]),
|
||||
Route("/api/v1/chats", create_chat, methods=["POST"]),
|
||||
Route("/api/v1/chats", list_chats, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
|
||||
]
|
||||
app = Starlette(routes=routes, lifespan=lifespan,
|
||||
exception_handlers={HTTPException: http_exception, Exception: server_error})
|
||||
app.add_middleware(RequestContextMiddleware)
|
||||
app.state.messenger = state
|
||||
return app
|
||||
|
||||
|
||||
__all__ = ["build_app"]
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Конфигурация из окружения. Секреты читаются из *_FILE (Docker/Compose secrets)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ConfigError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _secret(name: str, required: bool) -> str | None:
|
||||
"""Значение NAME или содержимое файла NAME_FILE (приоритет у файла)."""
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
try:
|
||||
value = Path(path).read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise ConfigError(f"cannot read {name}_FILE") from exc
|
||||
if not value:
|
||||
raise ConfigError(f"{name}_FILE is empty")
|
||||
return value
|
||||
value = os.environ.get(name, "").strip()
|
||||
if value:
|
||||
return value
|
||||
if required:
|
||||
raise ConfigError(f"{name} or {name}_FILE is required")
|
||||
return None
|
||||
|
||||
|
||||
def _int(name: str, default: int, lo: int, hi: int) -> int:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw == "":
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError as exc:
|
||||
raise ConfigError(f"{name} must be an integer") from exc
|
||||
if not lo <= value <= hi:
|
||||
raise ConfigError(f"{name} must be in [{lo}, {hi}]")
|
||||
return value
|
||||
|
||||
|
||||
def _float(name: str, default: float, lo: float, hi: float) -> float:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw == "":
|
||||
return default
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError as exc:
|
||||
raise ConfigError(f"{name} must be a number") from exc
|
||||
if not lo <= value <= hi:
|
||||
raise ConfigError(f"{name} must be in [{lo}, {hi}]")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
host: str
|
||||
port: int
|
||||
env: str
|
||||
database_url: str
|
||||
cache_url: str | None
|
||||
session_ttl_seconds: int
|
||||
session_cache_ttl_seconds: int
|
||||
cache_timeout_seconds: float
|
||||
cache_retry_seconds: float
|
||||
db_pool_min: int
|
||||
db_pool_max: int
|
||||
db_pool_timeout_seconds: float
|
||||
db_connect_timeout_seconds: int
|
||||
db_statement_timeout_ms: int
|
||||
request_timeout_seconds: float
|
||||
max_body_bytes: int
|
||||
kdf_concurrency: int
|
||||
scrypt_log2_n: int
|
||||
scrypt_r: int
|
||||
scrypt_p: int
|
||||
migrate_on_start: bool
|
||||
migrate_wait_seconds: int
|
||||
|
||||
@property
|
||||
def cache_enabled(self) -> bool:
|
||||
return self.cache_url is not None
|
||||
|
||||
|
||||
def load() -> Settings:
|
||||
store = os.environ.get("SESSION_STORE", "postgres").strip()
|
||||
if store != "postgres":
|
||||
# Источник истины сессий — всегда PostgreSQL; Redis/Valkey — только кэш (см. REPORT.md).
|
||||
raise ConfigError("SESSION_STORE must be 'postgres' (Redis/Valkey is a cache via CACHE_URL)")
|
||||
cache_mode = os.environ.get("SESSION_CACHE", "auto").strip()
|
||||
if cache_mode not in {"auto", "redis", "off"}:
|
||||
raise ConfigError("SESSION_CACHE must be auto, redis or off")
|
||||
cache_url = None if cache_mode == "off" else _secret("CACHE_URL", required=cache_mode == "redis")
|
||||
pool_max = _int("DB_POOL_MAX", 10, 1, 100)
|
||||
return Settings(
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0"),
|
||||
port=_int("APP_PORT", 8080, 1, 65535),
|
||||
env=os.environ.get("APP_ENV", "lab"),
|
||||
database_url=_secret("DATABASE_URL", required=True) or "",
|
||||
cache_url=cache_url,
|
||||
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
|
||||
session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60, 1, 3600),
|
||||
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.2, 0.01, 5),
|
||||
cache_retry_seconds=_float("CACHE_RETRY_SECONDS", 2.0, 0.1, 60),
|
||||
db_pool_min=min(_int("DB_POOL_MIN", 1, 0, 100), pool_max),
|
||||
db_pool_max=pool_max,
|
||||
db_pool_timeout_seconds=_float("DB_POOL_TIMEOUT_SECONDS", 2.0, 0.1, 60),
|
||||
db_connect_timeout_seconds=_int("DB_CONNECT_TIMEOUT_SECONDS", 3, 1, 60),
|
||||
db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000, 100, 600000),
|
||||
request_timeout_seconds=_float("REQUEST_TIMEOUT_SECONDS", 10.0, 1, 120),
|
||||
max_body_bytes=_int("MAX_BODY_BYTES", 65536, 1024, 1048576),
|
||||
kdf_concurrency=_int("KDF_CONCURRENCY", 4, 1, 64),
|
||||
scrypt_log2_n=_int("SCRYPT_LOG2_N", 15, 14, 20),
|
||||
scrypt_r=_int("SCRYPT_R", 8, 1, 32),
|
||||
scrypt_p=_int("SCRYPT_P", 3, 1, 16),
|
||||
migrate_on_start=os.environ.get("MIGRATE_ON_START", "true").lower() in {"1", "true", "yes"},
|
||||
migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60, 0, 600),
|
||||
)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Пул соединений PostgreSQL, версионируемые миграции и классификация ошибок БД."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool, PoolTimeout
|
||||
|
||||
from .config import Settings
|
||||
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
|
||||
MIGRATION_LOCK_ID = 2026_0202 # pg_advisory_lock: параллельные реплики не применяют миграции дважды
|
||||
|
||||
|
||||
def connection_kwargs(settings: Settings) -> dict:
|
||||
options = (
|
||||
f"-c statement_timeout={settings.db_statement_timeout_ms} "
|
||||
"-c lock_timeout=2000 "
|
||||
"-c idle_in_transaction_session_timeout=10000 "
|
||||
"-c TimeZone=UTC"
|
||||
)
|
||||
return {
|
||||
"connect_timeout": settings.db_connect_timeout_seconds,
|
||||
"application_name": "messenger-api",
|
||||
"options": options,
|
||||
"row_factory": dict_row,
|
||||
}
|
||||
|
||||
|
||||
def make_pool(settings: Settings) -> AsyncConnectionPool:
|
||||
return AsyncConnectionPool(
|
||||
conninfo=settings.database_url,
|
||||
kwargs=connection_kwargs(settings),
|
||||
min_size=settings.db_pool_min,
|
||||
max_size=settings.db_pool_max,
|
||||
timeout=settings.db_pool_timeout_seconds, # ожидание свободного соединения
|
||||
max_waiting=200, # очередь ожидания ограничена
|
||||
max_idle=300,
|
||||
reconnect_timeout=30,
|
||||
check=AsyncConnectionPool.check_connection, # после рестарта PG битые соединения отбрасываются
|
||||
open=False,
|
||||
name="messenger",
|
||||
)
|
||||
|
||||
|
||||
def is_unavailable(exc: BaseException) -> bool:
|
||||
"""Ошибки, означающие «зависимость недоступна/перегружена», а не баг приложения."""
|
||||
if isinstance(exc, (PoolTimeout, psycopg.OperationalError, asyncio.TimeoutError, ConnectionError)):
|
||||
return True
|
||||
return isinstance(exc, (pg_errors.QueryCanceled, pg_errors.LockNotAvailable,
|
||||
pg_errors.AdminShutdown, pg_errors.CannotConnectNow))
|
||||
|
||||
|
||||
def iso(ts: datetime) -> str:
|
||||
"""RFC 3339 UTC с микросекундами и суффиксом Z (точность timestamptz в PostgreSQL)."""
|
||||
return ts.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
|
||||
def _migration_files() -> list[tuple[str, str, str]]:
|
||||
result = []
|
||||
for path in sorted(MIGRATIONS_DIR.glob("[0-9][0-9][0-9][0-9]_*.sql")):
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
result.append((path.stem, sql, hashlib.sha256(sql.encode()).hexdigest()))
|
||||
return result
|
||||
|
||||
|
||||
async def migrate(settings: Settings) -> list[str]:
|
||||
"""Применяет недостающие миграции. Повторный запуск — no-op; изменённая применённая миграция —
|
||||
ошибка (миграции неизменяемы, исправления — новой версией)."""
|
||||
applied_now: list[str] = []
|
||||
async with await psycopg.AsyncConnection.connect(
|
||||
settings.database_url, autocommit=True, **connection_kwargs(settings)) as conn:
|
||||
await conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,))
|
||||
try:
|
||||
await conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY, checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT clock_timestamp())")
|
||||
rows = await (await conn.execute("SELECT version, checksum FROM schema_migrations")).fetchall()
|
||||
done = {r["version"]: r["checksum"] for r in rows}
|
||||
for version, sql, checksum in _migration_files():
|
||||
if version in done:
|
||||
if done[version] != checksum:
|
||||
raise RuntimeError(f"applied migration {version} was modified")
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
|
||||
(version, checksum))
|
||||
applied_now.append(version)
|
||||
log.info("migration applied", extra={"version": version})
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,))
|
||||
return applied_now
|
||||
|
||||
|
||||
async def migrate_with_wait(settings: Settings) -> list[str]:
|
||||
deadline = time.monotonic() + settings.migrate_wait_seconds
|
||||
delay = 0.5
|
||||
while True:
|
||||
try:
|
||||
return await migrate(settings)
|
||||
except psycopg.OperationalError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
log.warning("database not reachable yet; retrying migrations")
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, 5)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Единый формат ошибок API: {"error":{"code","message","request_id"}}."""
|
||||
from __future__ import annotations
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
_STATUS = {
|
||||
"invalid_request": 400,
|
||||
"unauthorized": 401,
|
||||
"not_found": 404,
|
||||
"method_not_allowed": 405,
|
||||
"conflict": 409,
|
||||
"payload_too_large": 413,
|
||||
"unsupported_media_type": 415,
|
||||
"rate_limited": 429,
|
||||
"internal_error": 500,
|
||||
"unavailable": 503,
|
||||
}
|
||||
|
||||
_DEFAULT_MESSAGE = {
|
||||
"invalid_request": "Некорректный запрос",
|
||||
"unauthorized": "Требуется действительная аутентификация",
|
||||
"not_found": "Объект не найден",
|
||||
"conflict": "Конфликт состояния",
|
||||
"payload_too_large": "Слишком большой запрос",
|
||||
"unsupported_media_type": "Неподдерживаемый тип содержимого",
|
||||
"rate_limited": "Слишком много запросов",
|
||||
"internal_error": "Внутренняя ошибка",
|
||||
"unavailable": "Сервис временно недоступен",
|
||||
}
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, code: str, message: str | None = None, *, challenge: str | None = None,
|
||||
headers: dict[str, str] | None = None):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.status = _STATUS[code]
|
||||
self.message = message or _DEFAULT_MESSAGE.get(code, "Ошибка")
|
||||
self.headers = dict(headers or {})
|
||||
if challenge:
|
||||
self.headers["WWW-Authenticate"] = challenge
|
||||
|
||||
@property
|
||||
def public_code(self) -> str:
|
||||
# 405 не входит в enum контракта: отдаём invalid_request с корректным HTTP-статусом.
|
||||
return "invalid_request" if self.code == "method_not_allowed" else self.code
|
||||
|
||||
|
||||
def invalid(message: str) -> ApiError:
|
||||
return ApiError("invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized_bearer() -> ApiError:
|
||||
return ApiError("unauthorized", challenge=BEARER_CHALLENGE)
|
||||
|
||||
|
||||
def unauthorized_basic() -> ApiError:
|
||||
return ApiError("unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE)
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
return ApiError("not_found")
|
||||
|
||||
|
||||
def unavailable() -> ApiError:
|
||||
return ApiError("unavailable", headers={"Retry-After": "1"})
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Password KDF (scrypt из OpenSSL через hashlib) и opaque-токены сессий.
|
||||
|
||||
Пароль хешируется медленным memory-hard KDF с солью: цель — сделать перебор украденных
|
||||
hash дорогим. Токен сессии уже содержит 256 бит случайности, перебирать его бессмысленно,
|
||||
поэтому для индекса в БД/кэше достаточно быстрого SHA-256 — это разные задачи.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
_PREFIX = "$scrypt$"
|
||||
|
||||
|
||||
def _b64(data: bytes) -> str:
|
||||
return base64.b64encode(data).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _unb64(text: str) -> bytes:
|
||||
return base64.b64decode(text + "=" * (-len(text) % 4), validate=True)
|
||||
|
||||
|
||||
class PasswordHasher:
|
||||
"""Формат PHC-подобный: $scrypt$ln=15,r=8,p=3$<salt>$<hash>. Параметры хранятся в строке,
|
||||
поэтому их можно усилить позднее без потери старых hash."""
|
||||
|
||||
def __init__(self, log2_n: int, r: int, p: int, concurrency: int):
|
||||
self.log2_n, self.r, self.p = log2_n, r, p
|
||||
# Ограничиваем одновременные KDF: каждый вызов занимает 128*r*N байт RAM (32 MiB при ln=15).
|
||||
self._slots = asyncio.Semaphore(concurrency)
|
||||
self._dummy: str | None = None
|
||||
|
||||
def _maxmem(self, log2_n: int, r: int, p: int) -> int:
|
||||
return 128 * r * (2 ** log2_n) * 2 + 1024 * 1024 * p
|
||||
|
||||
def _derive(self, password: str, salt: bytes, log2_n: int, r: int, p: int) -> bytes:
|
||||
return hashlib.scrypt(password.encode("utf-8"), salt=salt, n=2 ** log2_n, r=r, p=p,
|
||||
maxmem=self._maxmem(log2_n, r, p), dklen=32)
|
||||
|
||||
def hash_sync(self, password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = self._derive(password, salt, self.log2_n, self.r, self.p)
|
||||
return f"{_PREFIX}ln={self.log2_n},r={self.r},p={self.p}${_b64(salt)}${_b64(digest)}"
|
||||
|
||||
def verify_sync(self, password: str, encoded: str) -> bool:
|
||||
try:
|
||||
if not encoded.startswith(_PREFIX):
|
||||
return False
|
||||
params, salt_b64, hash_b64 = encoded[len(_PREFIX):].split("$")
|
||||
fields = dict(item.split("=", 1) for item in params.split(","))
|
||||
log2_n, r, p = int(fields["ln"]), int(fields["r"]), int(fields["p"])
|
||||
if not (10 <= log2_n <= 22 and 1 <= r <= 32 and 1 <= p <= 16):
|
||||
return False
|
||||
expected = _unb64(hash_b64)
|
||||
actual = self._derive(password, _unb64(salt_b64), log2_n, r, p)
|
||||
except (ValueError, KeyError):
|
||||
return False
|
||||
return hmac.compare_digest(actual, expected)
|
||||
|
||||
async def hash(self, password: str) -> str:
|
||||
async with self._slots:
|
||||
return await asyncio.to_thread(self.hash_sync, password)
|
||||
|
||||
async def verify(self, password: str, encoded: str | None) -> bool:
|
||||
"""Для неизвестного пользователя проверяем фиктивный hash: время ответа не выдаёт,
|
||||
существует ли username."""
|
||||
if encoded is None:
|
||||
if self._dummy is None:
|
||||
self._dummy = await self.hash("dummy-password-for-timing")
|
||||
encoded, password = self._dummy, password + "\x00never"
|
||||
async with self._slots:
|
||||
await asyncio.to_thread(self.verify_sync, password, encoded)
|
||||
return False
|
||||
async with self._slots:
|
||||
return await asyncio.to_thread(self.verify_sync, password, encoded)
|
||||
|
||||
|
||||
TOKEN_BYTES = 32 # 256 бит энтропии
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(TOKEN_BYTES) # 43 символа base64url
|
||||
|
||||
|
||||
def token_hash(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
|
||||
def looks_like_token(token: str) -> bool:
|
||||
return len(token) == 43 and all(c.isascii() and (c.isalnum() or c in "-_") for c in token)
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Сессии: PostgreSQL — источник истины, Redis/Valkey — read-through кэш с TTL.
|
||||
|
||||
Политика (подробно — REPORT.md):
|
||||
* ключ sess:v1:<sha256(token)> хранит {user, expires_at}; TTL = min(SESSION_CACHE_TTL_SECONDS,
|
||||
остаток жизни сессии) — кэш никогда не переживает срок сессии;
|
||||
* logout: revoked_at + строка outbox в одной транзакции PostgreSQL, затем в Redis атомарно
|
||||
«надгробие» sess:v1:revoked:<hash> и DEL ключа. Заполнение кэша — Lua SET … только если
|
||||
надгробия нет, поэтому конкурентный read-through не вернёт отозванную сессию в кэш;
|
||||
* Redis недоступен → аутентификация идёт в PostgreSQL (fail-open только к источнику истины,
|
||||
не к анонимному доступу); после ошибки кэш не читается, пока фоновый цикл не проверит Redis
|
||||
и не дообработает outbox инвалидаций;
|
||||
* flush/restart Redis = холодный кэш: сессии остаются в PostgreSQL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
from redis import asyncio as aioredis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .config import Settings
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
KEY = "sess:v1:"
|
||||
TOMBSTONE = "sess:v1:revoked:"
|
||||
|
||||
# KEYS[1]=session key, KEYS[2]=tombstone; ARGV[1]=value, ARGV[2]=ttl ms
|
||||
_SET_IF_NOT_REVOKED = """
|
||||
if redis.call('EXISTS', KEYS[2]) == 1 then return 0 end
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
|
||||
return 1
|
||||
"""
|
||||
# KEYS[1]=session key, KEYS[2]=tombstone; ARGV[1]=tombstone ttl ms
|
||||
_INVALIDATE = """
|
||||
redis.call('SET', KEYS[2], '1', 'PX', ARGV[1])
|
||||
redis.call('DEL', KEYS[1])
|
||||
return 1
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Principal:
|
||||
user: dict
|
||||
expires_at: float # epoch seconds
|
||||
token_hash: bytes
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.client: aioredis.Redis | None = None
|
||||
self._set_script = None
|
||||
self._invalidate_script = None
|
||||
# degraded=True: кэш не читаем (было исключение Redis или есть недообработанный outbox).
|
||||
self.degraded = True
|
||||
self._next_probe = 0.0
|
||||
self.stats = {"hits": 0, "misses": 0, "errors": 0}
|
||||
if settings.cache_url:
|
||||
t = settings.cache_timeout_seconds
|
||||
self.client = aioredis.from_url(
|
||||
settings.cache_url, socket_timeout=t, socket_connect_timeout=t,
|
||||
retry_on_timeout=False, health_check_interval=0, max_connections=50,
|
||||
decode_responses=False)
|
||||
self._set_script = self.client.register_script(_SET_IF_NOT_REVOKED)
|
||||
self._invalidate_script = self.client.register_script(_INVALIDATE)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.client is not None
|
||||
|
||||
def _tombstone_ms(self) -> int:
|
||||
# Надгробие живёт дольше любой записи кэша и любого запроса в полёте.
|
||||
return int((self.settings.session_cache_ttl_seconds + self.settings.request_timeout_seconds + 5) * 1000)
|
||||
|
||||
def _fail(self, op: str, exc: Exception) -> None:
|
||||
self.stats["errors"] += 1
|
||||
if not self.degraded:
|
||||
log.warning("cache unavailable; falling back to PostgreSQL",
|
||||
extra={"op": op, "error": type(exc).__name__})
|
||||
self.degraded = True
|
||||
self._next_probe = time.monotonic() + self.settings.cache_retry_seconds
|
||||
|
||||
async def get(self, th: bytes) -> Principal | None:
|
||||
if not self.client or self.degraded:
|
||||
return None
|
||||
try:
|
||||
raw = await self.client.get(KEY + th.hex())
|
||||
except (RedisError, OSError, asyncio.TimeoutError) as exc:
|
||||
self._fail("get", exc)
|
||||
return None
|
||||
if raw is None:
|
||||
self.stats["misses"] += 1
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
principal = Principal(user=data["user"], expires_at=float(data["exp"]), token_hash=th)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
if principal.expires_at <= time.time():
|
||||
return None
|
||||
self.stats["hits"] += 1
|
||||
return principal
|
||||
|
||||
async def put(self, p: Principal) -> None:
|
||||
if not self.client or self.degraded:
|
||||
return
|
||||
ttl_ms = int(min(self.settings.session_cache_ttl_seconds, p.expires_at - time.time()) * 1000)
|
||||
if ttl_ms < 1:
|
||||
return
|
||||
value = json.dumps({"user": p.user, "exp": p.expires_at}, separators=(",", ":"))
|
||||
try:
|
||||
await self._set_script(keys=[KEY + p.token_hash.hex(), TOMBSTONE + p.token_hash.hex()],
|
||||
args=[value, ttl_ms])
|
||||
except (RedisError, OSError, asyncio.TimeoutError) as exc:
|
||||
self._fail("put", exc)
|
||||
|
||||
async def invalidate(self, th: bytes) -> bool:
|
||||
if not self.client:
|
||||
return True
|
||||
try:
|
||||
await self._invalidate_script(keys=[KEY + th.hex(), TOMBSTONE + th.hex()],
|
||||
args=[self._tombstone_ms()])
|
||||
return True
|
||||
except (RedisError, OSError, asyncio.TimeoutError) as exc:
|
||||
self._fail("invalidate", exc)
|
||||
return False
|
||||
|
||||
async def ping(self) -> bool:
|
||||
if not self.client:
|
||||
return False
|
||||
try:
|
||||
return bool(await self.client.ping())
|
||||
except (RedisError, OSError, asyncio.TimeoutError):
|
||||
return False
|
||||
|
||||
async def drain_outbox(self, pool: AsyncConnectionPool) -> int:
|
||||
"""Повторяет инвалидации из session_cache_invalidations. True-результат: outbox пуст."""
|
||||
drained = 0
|
||||
while True:
|
||||
async with pool.connection() as conn:
|
||||
rows = await (await conn.execute(
|
||||
"SELECT token_hash FROM session_cache_invalidations ORDER BY created_at LIMIT 500"
|
||||
)).fetchall()
|
||||
if not rows:
|
||||
return drained
|
||||
done = []
|
||||
for row in rows:
|
||||
th = bytes(row["token_hash"])
|
||||
if not await self.invalidate(th):
|
||||
raise ConnectionError("cache still unavailable")
|
||||
done.append(th)
|
||||
async with pool.connection() as conn:
|
||||
await conn.execute("DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", (done,))
|
||||
drained += len(done)
|
||||
|
||||
async def maintenance_loop(self, pool: AsyncConnectionPool, stop: asyncio.Event) -> None:
|
||||
"""Возвращает кэш в работу только после успешного PING и пустого outbox."""
|
||||
while not stop.is_set():
|
||||
if self.client and time.monotonic() >= self._next_probe:
|
||||
try:
|
||||
if await self.ping():
|
||||
drained = await self.drain_outbox(pool)
|
||||
if self.degraded:
|
||||
log.info("cache available again", extra={"replayed_invalidations": drained})
|
||||
self.degraded = False
|
||||
else:
|
||||
self._fail("probe", ConnectionError("ping failed"))
|
||||
except ConnectionError as exc:
|
||||
self._fail("drain", exc)
|
||||
except Exception as exc: # noqa: BLE001 — БД недоступна: состояние кэша не меняем
|
||||
log.warning("outbox drain skipped", extra={"error": type(exc).__name__})
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=1.0)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.client:
|
||||
await self.client.aclose()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Строгая валидация входа: лишние поля, неверные типы и malformed JSON — 400."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from .errors import ApiError, invalid
|
||||
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
|
||||
|
||||
def _no_duplicates(pairs):
|
||||
obj = {}
|
||||
for key, value in pairs:
|
||||
if key in obj:
|
||||
raise ValueError("duplicate key")
|
||||
obj[key] = value
|
||||
return obj
|
||||
|
||||
|
||||
def _reject_constant(name):
|
||||
raise ValueError("non-standard JSON constant " + name)
|
||||
|
||||
|
||||
def parse_json_object(raw: bytes, allowed: set[str], required: set[str]) -> dict:
|
||||
if not raw:
|
||||
raise invalid("Ожидается JSON-объект")
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"), object_pairs_hook=_no_duplicates,
|
||||
parse_constant=_reject_constant)
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise invalid("Некорректный JSON") from None
|
||||
if not isinstance(data, dict):
|
||||
raise invalid("Ожидается JSON-объект")
|
||||
extra = set(data) - allowed
|
||||
if extra:
|
||||
raise invalid("Лишние поля: " + ", ".join(sorted(extra))[:200])
|
||||
missing = required - set(data)
|
||||
if missing:
|
||||
raise invalid("Отсутствуют поля: " + ", ".join(sorted(missing)))
|
||||
return data
|
||||
|
||||
|
||||
def text_field(data: dict, name: str, lo: int, hi: int) -> str:
|
||||
value = data.get(name)
|
||||
if not isinstance(value, str):
|
||||
raise invalid(f"Поле {name} должно быть строкой")
|
||||
if not lo <= len(value) <= hi: # len() в Python — число Unicode code points
|
||||
raise invalid(f"Поле {name}: длина {lo}–{hi} символов")
|
||||
if lo > 0 and not value.strip():
|
||||
raise invalid(f"Поле {name} не может состоять только из пробелов")
|
||||
if "\x00" in value:
|
||||
raise invalid(f"Поле {name} содержит недопустимый символ")
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError: # одиночные суррогаты из \uD800-escape
|
||||
raise invalid(f"Поле {name} содержит некорректный Unicode") from None
|
||||
return value
|
||||
|
||||
|
||||
def parse_uuid(value, what: str = "UUID") -> uuid.UUID:
|
||||
if not isinstance(value, str):
|
||||
raise invalid(f"Некорректный {what}")
|
||||
try:
|
||||
parsed = uuid.UUID(value)
|
||||
except ValueError:
|
||||
raise invalid(f"Некорректный {what}") from None
|
||||
# Принимаем только каноническую 36-символьную запись (без {} и urn:).
|
||||
if len(value) != 36:
|
||||
raise invalid(f"Некорректный {what}")
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_limit(params) -> int:
|
||||
values = params.getlist("limit")
|
||||
if not values:
|
||||
return 50
|
||||
if len(values) > 1 or not values[0].isascii() or not values[0].isdigit() or len(values[0]) > 3:
|
||||
raise invalid("limit должен быть целым 1–100")
|
||||
limit = int(values[0])
|
||||
if not 1 <= limit <= 100:
|
||||
raise invalid("limit должен быть целым 1–100")
|
||||
return limit
|
||||
|
||||
|
||||
def single_param(params, name: str) -> str | None:
|
||||
values = params.getlist(name)
|
||||
if len(values) > 1:
|
||||
raise invalid(f"Параметр {name} указан несколько раз")
|
||||
return values[0] if values else None
|
||||
|
||||
|
||||
# --- keyset cursor -------------------------------------------------------------------------
|
||||
# Курсор непрозрачен для клиента: base64url(JSON) с видом списка и областью (чат/пользователь).
|
||||
# Подделка курсора не даёт доступа: область всё равно проверяется, а позиция лишь фильтр.
|
||||
|
||||
def encode_cursor(kind: str, scope: uuid.UUID, created_at: datetime, item_id: uuid.UUID) -> str:
|
||||
payload = json.dumps({"v": 1, "k": kind, "s": str(scope), "t": created_at.isoformat(), "i": str(item_id)},
|
||||
separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, kind: str, scope: uuid.UUID) -> tuple[datetime, uuid.UUID]:
|
||||
try:
|
||||
if not cursor or len(cursor) > 512 or not cursor.isascii():
|
||||
raise ValueError
|
||||
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
|
||||
data = json.loads(raw)
|
||||
if data.get("v") != 1 or data.get("k") != kind or data.get("s") != str(scope):
|
||||
raise ValueError
|
||||
ts = datetime.fromisoformat(data["t"])
|
||||
if ts.tzinfo is None:
|
||||
raise ValueError
|
||||
return ts, uuid.UUID(data["i"])
|
||||
except (ValueError, KeyError, TypeError, AttributeError, UnicodeError):
|
||||
raise ApiError("invalid_request", "Некорректный cursor") from None
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Общая обвязка собственных проверок студента (stdlib, Python 3.9+).
|
||||
|
||||
Переиспользует HTTP-клиент публичных тестов (tests/client.py), не изменяя его. Команды управления
|
||||
стендом и доступа к SQL задаются переменными окружения; по умолчанию — Compose из этого репозитория.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
DC = os.environ.get("COMPOSE", "docker compose --env-file .env")
|
||||
PSQL = os.environ.get("PSQL", DC + " exec -T postgres psql -U messenger -d messenger")
|
||||
COMMANDS = {
|
||||
"stop_db": os.environ.get("STOP_DB", DC + " stop postgres"),
|
||||
"start_db": os.environ.get("START_DB", DC + " start postgres"),
|
||||
"stop_cache": os.environ.get("STOP_CACHE", DC + " stop redis"),
|
||||
"start_cache": os.environ.get("START_CACHE", DC + " start redis"),
|
||||
}
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080")
|
||||
TTL_BASE_URL = os.environ.get("TTL_BASE_URL", "http://localhost:8081")
|
||||
TTL_SECONDS = float(os.environ.get("TTL_SECONDS", "2"))
|
||||
|
||||
|
||||
def sh(name_or_cmd, timeout=90):
|
||||
cmd = COMMANDS.get(name_or_cmd, name_or_cmd)
|
||||
subprocess.run(shlex.split(cmd), cwd=ROOT, check=True, timeout=timeout,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def sql(query):
|
||||
"""Одна SQL-команда через psql (-At); возвращает строки результата. Только синтетические данные."""
|
||||
out = subprocess.run(shlex.split(PSQL) + ["-X", "-At", "-v", "ON_ERROR_STOP=1", "-c", query],
|
||||
cwd=ROOT, check=True, capture_output=True, text=True, timeout=60)
|
||||
return [line for line in out.stdout.splitlines() if line]
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()
|
||||
|
||||
|
||||
def new_user(c, label="s", password=None):
|
||||
username = ("st_" + label + "_" + uuid.uuid4().hex[:10])[:32]
|
||||
password = password or ("Pw-" + uuid.uuid4().hex)
|
||||
user = c.call("POST", "/api/v1/users", 201, schema="User",
|
||||
body={"username": username, "display_name": "Тест " + label, "password": password})["json"]
|
||||
return {"id": user["id"], "user": user, "username": username, "password": password}
|
||||
|
||||
|
||||
def login(c, actor, expected=201):
|
||||
r = c.call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(actor["username"], actor["password"])},
|
||||
schema="Session" if expected == 201 else None)
|
||||
if expected == 201:
|
||||
actor["token"] = r["json"]["token"]
|
||||
actor["expires_at"] = r["json"]["expires_at"]
|
||||
return r
|
||||
|
||||
|
||||
def wait_until(probe, timeout=60, interval=0.5):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if probe():
|
||||
return time.monotonic()
|
||||
except Failure as exc:
|
||||
last = exc
|
||||
time.sleep(interval)
|
||||
raise Failure("condition not reached in %ss (%s)" % (timeout, last))
|
||||
|
||||
|
||||
class Suite:
|
||||
def __init__(self, name):
|
||||
self.name, self.results = name, []
|
||||
|
||||
def case(self, title):
|
||||
def decorator(fn):
|
||||
self.results.append((title, fn))
|
||||
return fn
|
||||
return decorator
|
||||
|
||||
def run(self):
|
||||
passed = failed = 0
|
||||
print("## " + self.name)
|
||||
for title, fn in self.results:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
note = fn()
|
||||
passed += 1
|
||||
print("PASS %-62s %6.2fs%s" % (title, time.monotonic() - start, (" " + note) if note else ""))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
failed += 1
|
||||
print("FAIL %-62s %s" % (title, exc))
|
||||
if os.environ.get("DEBUG"):
|
||||
traceback.print_exc()
|
||||
print("summary: %d passed, %d failed" % (passed, failed))
|
||||
raise SystemExit(1 if failed else 0)
|
||||
|
||||
|
||||
__all__ = ["Client", "Failure", "check", "sh", "sql", "basic", "new_user", "login", "wait_until", "Suite",
|
||||
"BASE_URL", "TTL_BASE_URL", "TTL_SECONDS", "json"]
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Негативные и контрактные проверки уровня 3/4: аутентификация, KDF, валидация, ACL, пагинация,
|
||||
атомарность/rollback создания чата, гонка регистрации, параметризация SQL."""
|
||||
import hashlib
|
||||
import threading
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from common import BASE_URL, Client, Suite, basic, check, login, new_user, sql
|
||||
|
||||
c = Client(BASE_URL)
|
||||
s = Suite("API, auth, validation, ACL, transactions (" + BASE_URL + ")")
|
||||
alice = new_user(c, "alice"); login(c, alice)
|
||||
bob = new_user(c, "bob"); login(c, bob)
|
||||
eve = new_user(c, "eve"); login(c, eve)
|
||||
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat",
|
||||
body={"title": "student-tests", "member_ids": [alice["id"], bob["id"]]})["json"]
|
||||
CHAT = "/api/v1/chats/" + chat["id"]
|
||||
|
||||
|
||||
@s.case("password stored only as scrypt KDF hash; User has no secret fields")
|
||||
def _():
|
||||
row = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice["id"])[0]
|
||||
check(row.startswith("$scrypt$ln=15,r=8,p=3$"), "unexpected hash format")
|
||||
check(alice["password"] not in row, "plaintext password in DB")
|
||||
check(set(alice["user"]) == {"id", "username", "display_name", "created_at"}, "User leaks fields")
|
||||
other = new_user(c, "same", password=alice["password"])
|
||||
h2 = sql("SELECT password_hash FROM users WHERE id = '%s'" % other["id"])[0]
|
||||
check(h2 != row, "same password produced same hash: salt missing")
|
||||
|
||||
|
||||
@s.case("wrong password and unknown user: identical 401 + Basic challenge")
|
||||
def _():
|
||||
r1 = c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(alice["username"], "wrong-password-123")})
|
||||
r2 = c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("no_such_user_x", "wrong-password-123")})
|
||||
check(r1["json"]["error"]["message"] == r2["json"]["error"]["message"], "messages differ")
|
||||
check(r1["headers"]["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"', "challenge mismatch")
|
||||
|
||||
|
||||
@s.case("malformed Basic / Bearer on /auth/sessions / body on login")
|
||||
def _():
|
||||
for value in ("Basic", "Basic !!!", "Basic " + "bm9jb2xvbg==", "Bearer " + "x" * 43):
|
||||
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": value})
|
||||
c.call("POST", "/api/v1/auth/sessions", 401)
|
||||
c.call("POST", "/api/v1/auth/sessions", 400, body={"username": alice["username"]},
|
||||
headers={"Authorization": basic(alice["username"], alice["password"])})
|
||||
|
||||
|
||||
@s.case("password with ':' and non-ASCII UTF-8 works (RFC 7617)")
|
||||
def _():
|
||||
u = new_user(c, "colon", password="пароль:с:двоеточием-😀")
|
||||
login(c, u)
|
||||
me = c.call("GET", "/api/v1/users/me", actor=u, schema="User")["json"]
|
||||
check(me["id"] == u["id"], "wrong user")
|
||||
|
||||
|
||||
@s.case("opaque 256-bit token, unique per login, DB stores only SHA-256")
|
||||
def _():
|
||||
a, b = dict(alice), dict(alice)
|
||||
login(c, a); login(c, b)
|
||||
check(a["token"] != b["token"] and len(a["token"]) == 43, "token not unique/opaque")
|
||||
check(a["token"] not in (alice["id"], alice["username"]), "token derived from identity")
|
||||
digest = hashlib.sha256(a["token"].encode()).hexdigest()
|
||||
check(sql("SELECT count(*) FROM sessions WHERE token_hash = decode('%s','hex')" % digest) == ["1"], "hash not found")
|
||||
check(sql("SELECT count(*) FROM information_schema.columns WHERE table_name = 'sessions'"
|
||||
" AND data_type IN ('text', 'character varying')") == ["0"], "sessions has a text column for raw token")
|
||||
|
||||
|
||||
@s.case("only Bearer accepted; Basic and X-User-Id never authenticate")
|
||||
def _():
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": basic(alice["username"], alice["password"])})
|
||||
c.call("GET", "/api/v1/chats", 401, headers={"X-User-Id": alice["id"]})
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "A" * 43})
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer short"})
|
||||
me = c.call("GET", "/api/v1/users/me", actor=bob, headers={"X-User-Id": alice["id"]}, schema="User")["json"]
|
||||
check(me["id"] == bob["id"], "X-User-Id overrode session")
|
||||
|
||||
|
||||
@s.case("logout revokes only the current session; repeat -> 401")
|
||||
def _():
|
||||
u = new_user(c, "logout"); first = dict(u); second = dict(u)
|
||||
login(c, first); login(c, second)
|
||||
c.call("GET", "/api/v1/users/me", actor=first, schema="User") # прогреть кэш
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=first)
|
||||
for path in ("/api/v1/users/me", "/api/v1/chats"):
|
||||
c.call("GET", path, 401, actor=first)
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=first)
|
||||
c.call("GET", "/api/v1/users/me", actor=second, schema="User")
|
||||
check(sql("SELECT count(*) FROM sessions WHERE user_id='%s' AND revoked_at IS NOT NULL" % u["id"]) == ["1"],
|
||||
"revocation not persisted")
|
||||
|
||||
|
||||
@s.case("strict JSON validation -> 400")
|
||||
def _():
|
||||
base = {"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "X", "password": "p" * 12}
|
||||
bad = [dict(base, extra=1), dict(base, username="UPPER"), dict(base, username="ab"),
|
||||
dict(base, display_name=" "), dict(base, display_name="x" * 101), dict(base, password="p" * 11),
|
||||
dict(base, password="p" * 129), dict(base, display_name=5), {"username": base["username"]}]
|
||||
for body in bad:
|
||||
c.call("POST", "/api/v1/users", 400, body=body)
|
||||
c.call("POST", "/api/v1/users", 400, raw=b'{"username": ', headers={"Content-Type": "application/json"})
|
||||
c.call("POST", "/api/v1/users", 400, raw=b'[1,2]', headers={"Content-Type": "application/json"})
|
||||
c.call("POST", "/api/v1/users", 400, raw=b'{"username":"a","username":"b"}', headers={"Content-Type": "application/json"})
|
||||
c.call("POST", "/api/v1/users", 400, raw=b'username=x', headers={"Content-Type": "application/x-www-form-urlencoded"})
|
||||
c.call("POST", "/api/v1/users", 413, raw=b'{"x":"' + b"a" * 70000 + b'"}', headers={"Content-Type": "application/json"})
|
||||
|
||||
|
||||
@s.case("message text: 4000 code points OK, 4001 / whitespace / sender_id -> 400")
|
||||
def _():
|
||||
msg = c.message(alice, chat, "😀" * 4000)["json"]
|
||||
check(len(msg["text"]) == 4000, "text truncated")
|
||||
for body in ({"text": "a" * 4001}, {"text": " \n\t "}, {"text": "x", "sender_id": bob["id"]}, {"text": None}, {}):
|
||||
c.call("POST", CHAT + "/messages", 400, actor=alice, body=body)
|
||||
|
||||
|
||||
@s.case("SQL is parameterized: injection payloads stored verbatim")
|
||||
def _():
|
||||
payload = "'); DROP TABLE messages; -- \" \\ %s $1"
|
||||
msg = c.message(alice, chat, payload)["json"]
|
||||
check(msg["text"] == payload, "payload altered")
|
||||
c.call("GET", CHAT + "/messages?limit=1", actor=alice, schema="MessagePage")
|
||||
c.call("GET", "/api/v1/chats/" + quote("' OR 1=1 --", safe=""), 400, actor=alice)
|
||||
|
||||
|
||||
@s.case("ACL: outsider gets 404 everywhere (even for invalid body)")
|
||||
def _():
|
||||
c.call("GET", CHAT, 404, actor=eve)
|
||||
c.call("GET", CHAT + "/messages", 404, actor=eve)
|
||||
c.call("POST", CHAT + "/messages", 404, actor=eve, body={"text": "x"})
|
||||
c.call("POST", CHAT + "/messages", 404, actor=eve, body={"text": ""})
|
||||
c.call("GET", "/api/v1/chats/" + str(uuid.uuid4()), 404, actor=alice)
|
||||
c.call("GET", "/api/v1/chats/not-a-uuid", 400, actor=alice)
|
||||
got = c.call("GET", CHAT, actor=bob, schema="Chat")["json"]
|
||||
check(got["member_ids"] == [alice["id"], bob["id"]], "member order/content changed")
|
||||
|
||||
|
||||
@s.case("create chat validation: duplicates, caller missing, unknown member")
|
||||
def _():
|
||||
c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": "d", "member_ids": [alice["id"], alice["id"]]})
|
||||
c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": "d", "member_ids": [bob["id"]]})
|
||||
c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": "d", "member_ids": []})
|
||||
c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": " ", "member_ids": [alice["id"]]})
|
||||
|
||||
|
||||
@s.case("compound create-chat rolls back on FK failure (no orphan chat)")
|
||||
def _():
|
||||
title = "rollback-" + uuid.uuid4().hex
|
||||
before = sql("SELECT count(*) FROM chats")[0]
|
||||
c.call("POST", "/api/v1/chats", 400, actor=alice,
|
||||
body={"title": title, "member_ids": [alice["id"], bob["id"], str(uuid.uuid4())]})
|
||||
check(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title) == ["0"], "orphan chat committed")
|
||||
check(sql("SELECT count(*) FROM chat_members m LEFT JOIN chats c ON c.id = m.chat_id WHERE c.id IS NULL") == ["0"],
|
||||
"orphan membership")
|
||||
return "chats before=%s, failed insert left 0 rows" % before
|
||||
|
||||
|
||||
@s.case("registration race: 20 parallel same username -> exactly one 201")
|
||||
def _():
|
||||
username = "race_" + uuid.uuid4().hex[:12]
|
||||
statuses, lock = [], threading.Lock()
|
||||
barrier = threading.Barrier(20)
|
||||
|
||||
def worker():
|
||||
local = Client(BASE_URL, timeout=30)
|
||||
barrier.wait()
|
||||
r = local.call("POST", "/api/v1/users", [201, 409],
|
||||
body={"username": username, "display_name": "R", "password": "Race-password-123"})
|
||||
with lock:
|
||||
statuses.append(r["status"])
|
||||
threads = [threading.Thread(target=worker) for _ in range(20)]
|
||||
[t.start() for t in threads]; [t.join() for t in threads]
|
||||
check(sorted(statuses) == [201] + [409] * 19, "statuses: %s" % sorted(statuses))
|
||||
check(sql("SELECT count(*) FROM users WHERE username = '%s'" % username) == ["1"], "duplicate rows")
|
||||
return "201x1, 409x19, rows=1"
|
||||
|
||||
|
||||
@s.case("keyset pagination: 120 msgs, varying limit, no gaps/duplicates")
|
||||
def _():
|
||||
room = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat",
|
||||
body={"title": "pages", "member_ids": [alice["id"], bob["id"]]})["json"]
|
||||
path = "/api/v1/chats/" + room["id"] + "/messages"
|
||||
sent = [c.message(alice if i % 2 else bob, room, "m%03d" % i)["json"] for i in range(120)]
|
||||
got, cursor, limits, n = [], None, [7, 1, 50, 100, 3], 0
|
||||
while True:
|
||||
q = "?limit=%d" % limits[n % len(limits)] + ("&cursor=" + quote(cursor, safe="") if cursor else "")
|
||||
page = c.call("GET", path + q, actor=bob, schema="MessagePage")["json"]
|
||||
got.extend(page["items"]); cursor = page["next_cursor"]; n += 1
|
||||
if cursor is None:
|
||||
break
|
||||
check([m["id"] for m in got] == [m["id"] for m in sent], "order/completeness mismatch")
|
||||
first = c.call("GET", path + "?limit=2", actor=bob, schema="MessagePage")["json"]
|
||||
c.call("GET", CHAT + "/messages?cursor=" + quote(first["next_cursor"], safe=""), 400, actor=alice)
|
||||
for q in ("?limit=0", "?limit=101", "?limit=abc", "?limit=-1", "?limit=1&limit=2", "?cursor=garbage", "?cursor="):
|
||||
c.call("GET", path + q, 400, actor=bob)
|
||||
default = c.call("GET", path, actor=bob, schema="MessagePage")["json"]
|
||||
check(len(default["items"]) == 50 and default["next_cursor"], "default limit must be 50")
|
||||
return "%d pages" % n
|
||||
|
||||
|
||||
@s.case("chat list: only own chats, ordered, cursor bound to user")
|
||||
def _():
|
||||
u = new_user(c, "lister"); login(c, u)
|
||||
made = [c.call("POST", "/api/v1/chats", 201, actor=u, schema="Chat",
|
||||
body={"title": "c%d" % i, "member_ids": [u["id"]]})["json"]["id"] for i in range(3)]
|
||||
got, cursor = [], None
|
||||
while True:
|
||||
page = c.call("GET", "/api/v1/chats?limit=1" + ("&cursor=" + quote(cursor, safe="") if cursor else ""),
|
||||
actor=u, schema="ChatPage")["json"]
|
||||
got += [x["id"] for x in page["items"]]; cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
foreign = cursor
|
||||
check(got == made, "chat list mismatch")
|
||||
c.call("GET", "/api/v1/chats?cursor=" + quote(foreign, safe=""), 400, actor=alice)
|
||||
|
||||
|
||||
@s.case("read-after-write: confirmed message visible to next read")
|
||||
def _():
|
||||
m = c.message(bob, chat, "raw-" + uuid.uuid4().hex)["json"]
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
page = c.call("GET", CHAT + "/messages?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""),
|
||||
actor=alice, schema="MessagePage")["json"]
|
||||
items += page["items"]; cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
check(items[-1] == m, "last message is not the just-written one")
|
||||
|
||||
|
||||
@s.case("unknown route 404 / wrong method 405 in error format")
|
||||
def _():
|
||||
c.call("GET", "/api/v1/nope", 404)
|
||||
r = c.call("PUT", "/api/v1/users", 405, body={})
|
||||
check(r["json"]["error"]["code"] == "invalid_request", "405 code")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
s.run()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""In-process проверки политики кэша сессий против настоящего Redis/Valkey (нужен CACHE_URL[_FILE]).
|
||||
|
||||
Сценарий гонки: запрос A прочитал сессию из PostgreSQL ДО logout, а записать её в кэш пытается ПОСЛЕ
|
||||
logout. Без защиты отозванный токен жил бы в кэше до SESSION_CACHE_TTL_SECONDS. Защита — «надгробие»
|
||||
и атомарный Lua SET-if-not-revoked. Запуск в Compose: make test-cache-race.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql://unused") # БД этому тесту не нужна
|
||||
|
||||
from messenger.config import load # noqa: E402
|
||||
from messenger.sessions import KEY, TOMBSTONE, Principal, SessionCache # noqa: E402
|
||||
|
||||
|
||||
async def main():
|
||||
settings = load()
|
||||
if not settings.cache_enabled:
|
||||
raise SystemExit("CACHE_URL or CACHE_URL_FILE is required")
|
||||
cache = SessionCache(settings)
|
||||
cache.degraded = False
|
||||
user = {"id": "00000000-0000-4000-8000-000000000000", "username": "cache_test", "display_name": "t",
|
||||
"created_at": "2026-01-01T00:00:00.000000Z"}
|
||||
failures = 0
|
||||
|
||||
def report(ok, title, note=""):
|
||||
nonlocal failures
|
||||
failures += 0 if ok else 1
|
||||
print(("PASS " if ok else "FAIL ") + title + ((" " + note) if note else ""))
|
||||
|
||||
th = secrets.token_bytes(32)
|
||||
p = Principal(user=user, expires_at=time.time() + 3600, token_hash=th)
|
||||
await cache.put(p)
|
||||
report((await cache.get(th)) is not None, "read-through put makes entry readable")
|
||||
ttl = await cache.client.pttl(KEY + th.hex())
|
||||
report(0 < ttl <= settings.session_cache_ttl_seconds * 1000, "entry TTL capped by SESSION_CACHE_TTL_SECONDS",
|
||||
"pttl=%dms" % ttl)
|
||||
|
||||
await cache.invalidate(th) # logout
|
||||
report((await cache.get(th)) is None, "invalidate removes entry")
|
||||
await cache.put(p) # «опоздавший» read-through после logout
|
||||
report((await cache.client.exists(KEY + th.hex())) == 0, "late put after logout is refused by tombstone")
|
||||
tomb = await cache.client.pttl(TOMBSTONE + th.hex())
|
||||
report(tomb > settings.session_cache_ttl_seconds * 1000, "tombstone outlives any cache entry", "pttl=%dms" % tomb)
|
||||
|
||||
th2 = secrets.token_bytes(32)
|
||||
await cache.put(Principal(user=user, expires_at=time.time() + 1.2, token_hash=th2))
|
||||
ttl2 = await cache.client.pttl(KEY + th2.hex())
|
||||
report(0 < ttl2 <= 1200, "entry never outlives session expiry", "pttl=%dms" % ttl2)
|
||||
await asyncio.sleep(1.3)
|
||||
report((await cache.get(th2)) is None, "expired session not served from cache")
|
||||
|
||||
th3 = secrets.token_bytes(32)
|
||||
await cache.put(Principal(user=user, expires_at=time.time() + 3600, token_hash=th3))
|
||||
cache._fail("test", ConnectionError("simulated"))
|
||||
report((await cache.get(th3)) is None, "degraded mode: cache is not read until outbox drained")
|
||||
|
||||
await cache.client.delete(KEY + th.hex(), TOMBSTONE + th.hex(), KEY + th3.hex())
|
||||
await cache.close()
|
||||
print("summary: %s" % ("all passed" if not failures else "%d failed" % failures))
|
||||
raise SystemExit(1 if failures else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Истечение сессии при коротком SESSION_TTL_SECONDS (экземпляр api-ttl, профиль compose `ttl`).
|
||||
|
||||
Проверяет и путь через кэш Redis (запись кэша прогрета сразу после входа), и путь через PostgreSQL:
|
||||
TTL записи в кэше = min(SESSION_CACHE_TTL_SECONDS, остаток сессии), поэтому кэш не продлевает сессию.
|
||||
"""
|
||||
import datetime as dt
|
||||
import time
|
||||
|
||||
from common import TTL_BASE_URL, TTL_SECONDS, Client, Suite, check, login, new_user, sql
|
||||
|
||||
c = Client(TTL_BASE_URL)
|
||||
s = Suite("session expiry, SESSION_TTL_SECONDS=%g (%s)" % (TTL_SECONDS, TTL_BASE_URL))
|
||||
|
||||
|
||||
@s.case("expires_at = now + SESSION_TTL_SECONDS")
|
||||
def _():
|
||||
u = new_user(c, "ttl"); login(c, u)
|
||||
left = dt.datetime.fromisoformat(u["expires_at"][:-1] + "+00:00") - dt.datetime.now(dt.timezone.utc)
|
||||
check(0 < left.total_seconds() <= TTL_SECONDS + 0.5, "unexpected TTL: %s" % left)
|
||||
return "remaining %.2fs" % left.total_seconds()
|
||||
|
||||
|
||||
@s.case("token valid before expiry, 401 Bearer after (cache warmed)")
|
||||
def _():
|
||||
u = new_user(c, "ttl2"); login(c, u)
|
||||
c.call("GET", "/api/v1/users/me", actor=u, schema="User") # заполняет кэш
|
||||
c.call("GET", "/api/v1/chats", actor=u, schema="ChatPage")
|
||||
time.sleep(TTL_SECONDS + 1.0)
|
||||
r = c.call("GET", "/api/v1/users/me", 401, actor=u)
|
||||
check(r["headers"]["WWW-Authenticate"].startswith("Bearer"), "challenge")
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=u)
|
||||
rows = sql("SELECT (expires_at < clock_timestamp())::text, (revoked_at IS NULL)::text FROM sessions s"
|
||||
" JOIN users u ON u.id = s.user_id WHERE u.username = '%s'" % u["username"])
|
||||
check(rows == ["true|true"], "expiry must come from expires_at, not deletion/revocation: %s" % rows)
|
||||
|
||||
|
||||
@s.case("re-login after expiry issues a new valid session")
|
||||
def _():
|
||||
u = new_user(c, "ttl3"); login(c, u)
|
||||
old = u["token"]
|
||||
time.sleep(TTL_SECONDS + 1.0)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=u)
|
||||
login(c, u)
|
||||
check(u["token"] != old, "token reused")
|
||||
c.call("GET", "/api/v1/users/me", actor=u, schema="User")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
s.run()
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отказы зависимостей: Redis (кэш) и PostgreSQL (источник истины).
|
||||
|
||||
ВНИМАНИЕ: тест останавливает и снова запускает контейнеры redis и postgres (по умолчанию через
|
||||
docker compose; команды переопределяются STOP_DB/START_DB/STOP_CACHE/START_CACHE). Volumes не трогает.
|
||||
"""
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from common import BASE_URL, Client, Failure, Suite, basic, check, login, new_user, sh, sql, wait_until
|
||||
|
||||
c = Client(BASE_URL, timeout=10)
|
||||
s = Suite("dependency failures (" + BASE_URL + ")")
|
||||
alice = new_user(c, "fa"); login(c, alice)
|
||||
bob = new_user(c, "fb"); login(c, bob)
|
||||
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat",
|
||||
body={"title": "failures", "member_ids": [alice["id"], bob["id"]]})["json"]
|
||||
PATH = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
|
||||
|
||||
def timed(fn):
|
||||
t = time.monotonic(); r = fn(); return r, time.monotonic() - t
|
||||
|
||||
|
||||
@s.case("Redis down: API bypasses cache via PostgreSQL, no anonymous trust")
|
||||
def _():
|
||||
c.call("GET", "/api/v1/users/me", actor=alice, schema="User") # запись в кэше есть
|
||||
victim = new_user(c, "fv"); login(c, victim)
|
||||
c.call("GET", "/api/v1/users/me", actor=victim, schema="User")
|
||||
sh("stop_cache")
|
||||
try:
|
||||
_, took = timed(lambda: c.call("GET", "/api/v1/users/me", actor=alice, schema="User"))
|
||||
check(took < 2.0, "fallback too slow: %.2fs" % took)
|
||||
c.call("GET", "/health/ready", schema="Health") # кэш не критичен
|
||||
c.message(alice, chat, "while redis is down")
|
||||
fresh = new_user(c, "fn"); login(c, fresh) # вход без Redis
|
||||
c.call("GET", "/api/v1/users/me", actor=fresh, schema="User")
|
||||
c.call("GET", "/api/v1/users/me", 401) # аноним — 401
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "B" * 43})
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=victim) # logout при мёртвом Redis
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=victim)
|
||||
pending = sql("SELECT count(*) FROM session_cache_invalidations")[0]
|
||||
finally:
|
||||
sh("start_cache")
|
||||
# После возврата Redis старая запись кэша victim не должна «воскресить» отозванный токен.
|
||||
wait_until(lambda: sql("SELECT count(*) FROM session_cache_invalidations") == ["0"], timeout=30)
|
||||
for _ in range(10):
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=victim)
|
||||
time.sleep(0.3)
|
||||
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
return "fallback latency %.3fs; outbox rows during outage=%s, drained after recovery" % (took, pending)
|
||||
|
||||
|
||||
@s.case("PostgreSQL down: bounded 503 in error format; anonymous still 401")
|
||||
def _():
|
||||
before = len(c.call("GET", PATH + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"])
|
||||
sh("stop_db")
|
||||
observed = {}
|
||||
try:
|
||||
c.call("GET", "/health/live", schema="Health")
|
||||
r, took = timed(lambda: c.call("GET", "/health/ready", 503))
|
||||
observed["ready"] = round(took, 2)
|
||||
for name, call in {
|
||||
"post_message": lambda: c.call("POST", PATH, 503, actor=alice, body={"text": "db down"}),
|
||||
"history": lambda: c.call("GET", PATH, 503, actor=bob),
|
||||
"login": lambda: c.call("POST", "/api/v1/auth/sessions", 503,
|
||||
headers={"Authorization": basic(alice["username"], alice["password"])}),
|
||||
"register": lambda: c.call("POST", "/api/v1/users", 503,
|
||||
body={"username": "x" + uuid.uuid4().hex[:10], "display_name": "x", "password": "p" * 12}),
|
||||
}.items():
|
||||
r, took = timed(call)
|
||||
check(r["json"]["error"]["code"] == "unavailable", name + ": wrong code")
|
||||
check(took < 5.0, name + ": not bounded (%.2fs)" % took)
|
||||
check("psycopg" not in r["json"]["error"]["message"].lower(), "leaks internals")
|
||||
observed[name] = round(took, 2)
|
||||
c.call("GET", "/api/v1/users/me", 401) # без токена — 401
|
||||
r = c.call("GET", "/api/v1/chats", [401, 503], headers={"Authorization": "Bearer " + "C" * 43})
|
||||
check(r["status"] != 200, "unknown token accepted")
|
||||
finally:
|
||||
sh("start_db")
|
||||
t0 = time.monotonic()
|
||||
wait_until(lambda: c.call("GET", "/health/ready", [200, 503])["status"] == 200, timeout=90)
|
||||
recovered = time.monotonic() - t0
|
||||
after = c.call("GET", PATH + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
|
||||
check(len(after) == before, "data changed across DB outage")
|
||||
c.message(alice, chat, "after db recovery")
|
||||
return "latency(s)=%s; ready again in %.1fs" % (observed, recovered)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
s.run()
|
||||
except Failure as exc:
|
||||
print("FAIL", exc)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user