56 changed files with 15111 additions and 39 deletions
+13
View File
@@ -18,3 +18,16 @@ __pycache__
node_modules
evidence
.DS_Store
**/__pycache__
**/*.py[cod]
.work
.pytest_cache
.ruff_cache
tests
student-tests
research
scripts
contracts
*.md
Makefile
compose.yaml
+1
View File
@@ -3,6 +3,7 @@
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
PUBLIC_PORT=8082
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
+18 -14
View File
@@ -1,18 +1,22 @@
## Лабораторная 2
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Маслов / БВТ2301
Целевая оценка (3/4/5): 5
Трек 5: трек 2, планы запросов и восстановление
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Перенесён собственный HTTP API ЛР1; добавлены PostgreSQL/SQL-миграции,
Argon2id, постоянные Bearer-сессии, Valkey cache, bounded pool/timeouts.
Запуск: uv sync --locked; make bootstrap; make up; make test test-student.
Проверки: [REPORT.md](../REPORT.md), evidence/clean-clone.json;
80 HTTP-тестов, реальные отказы/rollback, 101110 сообщений и backup/restore.
Ограничения: один host, без HA/TLS; snapshot не хранит поздние COMMIT.
Источники: контракт, PostgreSQL/Psycopg/Argon2/Valkey documentation.
Помощь ИИ использована при разборе требований, реализации и проверках;
результаты подтверждаются сохранёнными запусками.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+26
View File
@@ -0,0 +1,26 @@
name: Application checks
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-24.04
env:
NO_PROXY: localhost,127.0.0.1
no_proxy: localhost,127.0.0.1
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: '3.14.7'
- run: python -m pip install uv==0.13.7
- run: uv sync --locked
- run: make bootstrap lint check validate PYTHON=.venv/bin/python
- run: make up
- run: make test test-student
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
- run: make verify VERIFY_OUTPUT=.work/ci-lifecycle.json
- if: always()
run: docker compose down --volumes
+3
View File
@@ -28,5 +28,8 @@ coverage/
.DS_Store
.idea/
.vscode/
.pytest_cache/
.ruff_cache/
.work/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
+1
View File
@@ -0,0 +1 @@
3.14.7
+22
View File
@@ -0,0 +1,22 @@
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=uv /uv /usr/local/bin/uv
WORKDIR /app
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
COPY pyproject.toml uv.lock ./
RUN uv sync --locked --no-dev --no-install-project
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ARG VCS_REF=unknown
LABEL org.opencontainers.image.revision=$VCS_REF
WORKDIR /app
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY --from=builder /app/.venv /app/.venv
COPY src /app/src
COPY migrations /app/migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+25 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8082
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -24,3 +24,27 @@ load:
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
UV ?= uv
IMAGE ?= mtusi-messenger:lab2
VERIFY_OUTPUT ?= evidence/lifecycle-repeat.json
RESEARCH_OUTPUT ?= evidence/query-plan-repeat.json
.PHONY: bootstrap up down lint test-student image verify research
bootstrap:
$(PYTHON) scripts/bootstrap.py
up: bootstrap
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
down:
docker compose down
image:
docker build --build-arg VCS_REF=$$(git rev-parse HEAD) -t "$(IMAGE)" .
lint:
$(UV) run --locked ruff format src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py
$(UV) run --locked ruff check src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py
test-student:
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
verify:
$(UV) run --locked python -m scripts.verify_lifecycle --output "$(VERIFY_OUTPUT)"
research:
$(UV) run --locked python -m research.query_plan --output "$(RESEARCH_OUTPUT)"
+40
View File
@@ -0,0 +1,40 @@
# ЛР2 на 5 — план и гипотеза
Автор: Маслов, БВТ2301. Исходная ветка: master.
Перенесена только собственная реализация ЛР1, без её .git/contracts/tests.
## Архитектура до реализации запросов
API → сервисы → PostgreSQL repository, ограниченный sync pool.
Блокирующие сервисные операции выполняются в threadpool, не в event loop.
PostgreSQL — источник истины для users/chats/members/messages/sessions.
Valkey — disposable cache неизменяемого User, TTL30s; сессия/expiry/отзыв
всегда проверяются PostgreSQL. Отказ cache обходится; отказ PG даёт 503.
Таблицы: users(id PK, username UNIQUE, display_name, password_hash, time);
chats(id PK,title,time); members(chat,user PK/FK,position UNIQUE);
messages(id PK,chat,sender,text,time) с FK(chat,sender)→members;
sessions(token_digest PK,user FK,expires_at).
Индекс истории(chat_id,created_at,id), индекс members(user_id,chat_id).
UserDTO не содержит password_hash; raw session token никогда не хранится.
Транзакции: регистрация — одна INSERT с UNIQUE; чат+участники — одна;
сообщение — одна INSERT с FK; сессия и logout — отдельные durable записи.
Миграции версионируются SQL+checksum и блокируются advisory lock.
Не изменять применённые миграции. Compose запускает migrate перед API.
## Проверки и трек 2
Smoke, persistence с тем же Bearer, регистрационная гонка/rollback,
expiry2s на отдельном API, logout и неверный пароль, остановка cache/PG,
повторный migrate и bootstrap. Независимый clean clone и bundle.
Исследование на отдельной БД: >=100k сообщений в неравномерных чатах;
EXPLAIN ANALYZE BUFFERS после ANALYZE/прогрева, сравнение chat_id-only
и составного keyset-индекса, стоимость записи, WAL/размер индекса.
Backup pg_dump → новая БД pg_restore → count и выбранные полные строки.
Гипотеза до измерения: составной индекс уменьшает чтение середины
большого чата, но увеличивает размер/стоимость записи; величина неизвестна.
Дамп — snapshot, записи после snapshot без WAL архива могут потеряться.
Не выдавать SQL timings за HTTP latency и локальный backup за HA.
+254 -24
View File
@@ -1,37 +1,267 @@
# Отчёт — лабораторная 2
# Лабораторная 2 — PostgreSQL, сессии и Valkey
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Автор: **Маслов, БВТ2301**. Цель: **5**, трек 2 «План запроса и восстановление».
Ветка `solution/bvt2301-maslov`; основа — skeleton ЛР2, ветка master.
Публикации и удалённого CI пока нет.
## Запуск из чистого clone
## Воспроизводимый запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker 25+ с BuildKit/Compose v2+, make, Git и uv 0.13.7.
Python 3.14.7; FastAPI 0.143.3, Pydantic 2.14.1, Uvicorn 0.54.0,
psycopg 3.3.6, psycopg-pool 3.3.3, argon2-cffi 25.1.0, redis client 8.1.0.
Точные транзитивные версии — `uv.lock`. Базовые образы Python/uv,
PostgreSQL 18.3 и Valkey 8.1.3 закреплены тегами и digest.
## Архитектура и решения
```sh
uv sync --locked
make bootstrap
make check validate PYTHON=.venv/bin/python
make lint
make up
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 make test
make test-student
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make verify
uv run --locked python -m scripts.verify_pool --output evidence/pool-repeat.json
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
API: `http://127.0.0.1:8082`. PostgreSQL/Valkey не публикуют host-порты.
Для другого порта задать PUBLIC_PORT в окружении и передавать BASE_URL
проверкам. Все повторные evidence должны иметь новые имена файлов;
для make доступны VERIFY_OUTPUT и RESEARCH_OUTPUT.
## Доказательства критериев
`make down` останавливает стенд, сохраняя PostgreSQL volume. Его повторный
запуск использует прежние данные и секреты. Намеренное удаление **только
учебных данных данного проекта**: `docker compose down --volumes`.
Не применять к рабочим данным и не запускать глобальный Docker prune.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Bootstrap создаёт только отсутствующие `.env`/secret files. Каталог secrets
имеет 0700, файлы доступны non-root контейнеру через read-only file mounts.
Не пересоздавать пароль при сохранённом PG volume: initdb уже создал роль
со старым паролем. В Git/образ секреты и .env не попадают.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Конфигурация: APP_HOST/APP_PORT — bind внутри API; PUBLIC_PORT — Compose;
DATABASE_URL_FILE/CACHE_URL_FILE — пути к файловым секретам;
SESSION_TTL_SECONDS — абсолютный срок сессии (default 3600);
DB_POOL_SIZE — 1…32 (default 10). SESSION_STORE=postgres и APP_ENV в example
информационные: реализация не переключает источник истины этими строками.
Compose передаёт приложению только нужные ему настройки и секреты.
## Эксперименты и отказы
## Архитектура и перенос
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Из своей ЛР1 перенесены HTTP-контракт, строгие схемы, ошибки/request ID,
ACL, UUID/UTC, cursor и собственные HTTP-тесты. Сохранены неизменёнными
README/COURSE/contracts/tests **текущего** skeleton. Git-история ЛР1
не переносилась; репозиторий ЛР2 независим.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
```mermaid
flowchart LR
C[HTTP клиент] --> API[FastAPI: JSON, auth, request ID]
API --> T[Ограниченный threadpool: сервисные операции]
T --> PG[(PostgreSQL: данные и сессии)]
T --> V[(Valkey: карточка User, TTL 30 s)]
M[Одноразовый migrate] --> PG
```
## Ограничения и следующие шаги
SQL parameter binding используется для всех данных клиента. Динамические
идентификаторы в research — через psycopg.sql.Identifier и проверенный
UUID-префикс; в рабочем API пользователь не задаёт имена таблиц/SQL.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Модель и транзакции заданы до SQL в [PLAN.md](PLAN.md):
| Объект/операция | Ограничения и граница |
|---|---|
| users | UUID PK, UNIQUE username; KDF до получения соединения; INSERT атомарен |
| chats + members | FK, уникальный участник/позиция; чат и все участники в одной транзакции |
| messages | FK(chat_id,sender_id) на members; INSERT подтверждается после COMMIT |
| sessions | PK digest32bytes, FK user, created_at/expires_at; raw token не хранится |
| history | `(chat_id,created_at,id)`, tuple keyset + LIMIT limit+1 |
| migrations | SQL-файлы, checksum и advisory lock; отдельный сервис до API |
Миграции с нуля и повторное применение не создают пользовательские данные.
Изменение применённого файла отклоняется; добавляется новая миграция.
API сам схему не создаёт. Отказ migrate блокирует запуск API в Compose.
Пул на процесс: min 1 / max 10, ожидание 1 с; connect_timeout 2 с;
statement_timeout 2.5 с, lock_timeout 1 с, idle transaction timeout 6 с.
Общий клиентский срок операций на выданном соединении — 4 с, включая
замороженный сервер. При таймауте соединение закрывается и не возвращается
в пул как исправное. SQL, KDF и cache I/O не блокируют ASGI event loop.
Liveness не обращается к БД; readiness проверяет PostgreSQL, не Valkey.
## Пароли, сессии и кэш
- Argon2id: 64 MiB, t=3, p=4. Одновременно не больше двух KDF на процесс,
ожидание слота ограничено 5 с. UTF-8 пароль 12–128 code points обрабатывается
целиком, без усечения/нормализации. Проверено 127 emoji и двоеточие.
- Basic — только POST auth/sessions; собственный строгий base64/UTF-8 parser,
разделение по первому двоеточию. Неизвестный username проходит dummy KDF,
возвращается тот же 401. Хэш/пароль не входят в UserDTO и HTTP-ошибки.
- Токен — secrets.token_urlsafe(32), 256 бит энтропии; PostgreSQL хранит
только SHA-256 digest случайного токена. Это не password KDF и не JWT.
- Каждый запрос проверяет сессию и абсолютный expires_at **в PostgreSQL**.
Logout удаляет сессию; прежний cache hit не оживляет её. X-User-Id
игнорируется, Basic на бизнес-endpoint даёт 401. WWW-Authenticate
соответствует Basic/Bearer. Restart срок не продлевает.
- Valkey хранит только неизменяемую карточку User, TTL 30 с, maxmemory 64 MiB,
allkeys-lru; persistence выключена, собственного volume нет. Ошибка
cache — miss и чтение PG, backoff 5 с; socket/connect timeout 0.15 с,
скрытые retry отключены. При отказе PG cached User не авторизует запрос.
- API не изменяет карточки пользователей, поэтому инвалидации при записи
не требуется. При восстановлении старого snapshot вне API кэш надо
очистить. История вообще не кэшируется: read-after-write идёт из PG.
Образы API: non-root UID10001, exec Python PID1, healthcheck, read-only rootfs,
tmpfs и cap_drop=ALL. Лимиты API 1 CPU/512 MiB, PG 1 CPU/512 MiB, cache
0.25 CPU/128 MiB. Логи JSON: request ID/маршрут/код/время; без тел,
Authorization, паролей, токенов, их digest и параметров SQL.
## Таблица доказательств
| Критерий README | Команда | Факт / evidence |
|---|---|---|
| API ЛР1 + новый auth | `make test`, `make test-student` | Smoke PASS, 80 passed; [clean-clone-pre-pr.json](evidence/clean-clone-pre-pr.json) |
| Данные и прежний токен после restart всех stores | `make test-persistence`, `make verify` | [public-persistence.txt](evidence/public-persistence.txt), [lifecycle.json](evidence/lifecycle.json); CID/PID/StartedAt до/после |
| PK/FK/UNIQUE и атомарный чат | migrations, регистрационная гонка, injected rollback | [001_core.sql](migrations/001_core.sql), [lifecycle.json](evidence/lifecycle.json); после сбоя строка чата отсутствует |
| Версионируемые миграции и bootstrap | `make verify`, clean clone | Повтор без изменений, секреты сохранены, checksum mismatch отклонён; [clean-clone.json](evidence/clean-clone.json) |
| Реальная роль Valkey | `make verify` | 20 cache hits, stop/flush не теряют данные/сессию |
| Parameter binding, bounded pool и timeouts | `python -m scripts.verify_pool`, `make verify` | [pool.json](evidence/pool.json): pool1, ожидание 1.005 с →503; pause PG4.011 с →503 |
| Индекс, EXPLAIN ≥10k | `make research` | 90 планов для 5 размеров чата; [query-plan.json](evidence/query-plan.json) |
| Expiry, logout, неверный пароль, отказы | HTTP-тесты, lifecycle | TTL2s →401; stop cache →200; stop PG →503; logout →401 |
| Трек2: ≥100k, индексы/запись/backup | research | 101110 сообщений, 2 индекса, 3 повтора, восстановление и сверка |
| Независимый clean clone | `python scripts/verify_clean.py` | Полный PASS с новыми секретами/volumes, 80 passed; [clean-clone-pre-pr.json](evidence/clean-clone-pre-pr.json) |
Первая проверка не утверждает наличие карточки в cache непосредственно
перед PG fault без дополнительного EXISTS: [границы evidence](evidence/verification-notes.md).
Финальный вариант проверил это и неизменность expires_at.
## Финальный чистый clone
Последняя проверка перед PR: source commit
`5e7c93a0362e667551c8807a674fe04927b20e53`.
Новые `.env`, секреты и volumes; pinned image build; check/validate/lint;
публичный smoke; **80 passed за 93.23 с**; public persistence; lifecycle,
pool и checksum guard — PASS. Повторно выполнено исследование: 101110
сообщений, 90 EXPLAIN-планов, backup/restore snapshot совпал.
Хэши runtime-модулей совпали; image revision соответствует source commit.
Все временные containers/volumes этого clone удалены, рабочий volume сохранён.
Первая попытка clean-clone проверки остановилась на persistence: Docker
сменил автоматически назначенный host-порт при restart, а тест сохранил
прежний BASE_URL. Смена подтверждена на том же контейнере (65424 → 65441).
Проверка исправлена выбирать свободный **фиксированный** номер порта один
раз; код API/сессий не менялся. Первая попытка не выдана за успешную:
[описание](evidence/clean-attempt1.txt), [диагностика](evidence/port-restart.json).
Прежний полный повтор — [clean-clone.json](evidence/clean-clone.json).
Последний повтор перед PR —
[clean-clone-pre-pr.json](evidence/clean-clone-pre-pr.json).
После проверенного source commit менялись только отчёт, шаблон и evidence;
runtime src, зависимости, Dockerfile и Compose остались теми же.
Добавлен отдельный application workflow для CI; наличие файла не означает
его запуска на Gitea. Публикация, fork/MR и удалённый CI пока не выполнялись.
Повторная сверка выполнена по текущим README, COURSE, contracts/tests
и handoff от 30.10.2026. Все уровни 3/4 и выбранный трек 2 покрыты
таблицей доказательств. Исходные задание, контракт и публичные тесты
сохранены. Исправлены два пограничных дефекта: JSON принимается только
в UTF-8, десятичный `limit=0001` допустим. Три регрессионных теста сначала
упали на прежнем образе, затем прошли в новом чистом clone:
[исходные отказы](evidence/pre-pr-regressions.json).
При повторе также проверены неизменность абсолютного срока сессии,
наличие User в cache перед каждым отказом PG (503 за 4.044/0.011 с),
pool timeout 1.006 с, rollback, checksum миграции и backup/restore.
Исследовательские числа ниже относятся к исходному сохранённому
query-plan.json; новый clean clone независимо повторил методику и сверку,
но его timings не подставлялись в старые таблицы.
## Исследование: методика и результаты
Гипотеза в PLAN закоммичена до замера. Стенд: Apple M1, 8 CPU, 8 GiB RAM,
Docker Desktop 4.98.0/Engine29.9.2; PostgreSQL в контейнере 1 CPU/512 MiB.
Рабочие таблицы messenger не меняются: создаются research_UUID и restored_UUID
именно данного запуска, затем удаляются вместе с временным dump.
SQL seed: 5 чатов 90000/10000/1000/100/10, всего101110 сообщений;
уникальные тексты80ASCII. Seed обходит HTTP, это явно исследовательский
набор, а не нагрузка регистрации/Argon2. Схема и migration metadata
соответствуют приложению; исследовательский пользователь не используется
для входа. Данных других людей нет.
Сравниваются chat_id-only и `(chat_id,created_at,id)`. На каждом варианте
VACUUM ANALYZE и прогрев; head/middle/tail каждой истории, LIMIT 51,
EXPLAIN ANALYZE BUFFERS. Три повтора с порядком A/B, B/A, A/B; custom plans,
prepared statements в эксперименте отключены. Подготовительный OFFSET
применяется только для получения контрольной границы вне измеряемого
запроса; рабочая пагинация использует только keyset.
| Индекс | SQL middle90k, мс | Буферы | Размер, bytes | INSERT+COMMIT1000, мс | WAL, bytes |
|---|---:|---:|---:|---:|---:|
| chat_id | 7.237 | 2209 | 729088 | 37.53 | 347536 |
| keyset | 0.017 | 5 | 5980160 | 11.55 | 427672 |
Это медианы трёх повторов; min/max и остальные размеры/позиции —
[query-summary.csv](evidence/query-summary.csv), полные планы —
[query-plan.json](evidence/query-plan.json). График построен из raw и проверен:
![SQL планы](evidence/query-plans.png)
Гипотеза чтения подтверждается. Цена — индекс примерно в 8.2 раза больше
и больший WAL на запись. Рост именно времени INSERT **не подтвердился**:
в этом опыте keyset быстрее и на записи. Причину этого различия отдельный
профиль не устанавливал. WAL — глобальный PG-счётчик; фон и cache state
могут влиять. EXPLAIN execution time не называется HTTP latency или CPU time.
Backup pg_dump -Fc: 3026259 bytes, 0.480 с; restore в новую БД: 0.664 с.
Сверены count всех 6 таблиц, SHA-256 всей истории и 15 полных сообщений.
После snapshot специально сделан ещё один SQL COMMIT: в источнике 101111,
в восстановленной копии 101110. Это демонстрация границы snapshot/RPO,
не измерение потери HTTP-подтверждений и не универсальный RTO сервиса.
Пик PG cgroup с момента старта контейнера 266039296 bytes (включает подготовку
и прочие проверки); это не RSS отдельного процесса и не пик только SELECT.
```sh
uv run --locked python -m research.query_plan --output evidence/query-plan-repeat.json
uv run --locked python -m research.analyze evidence/query-plan-repeat.json
```
Запускать последовательно с проверками отказов. Сырые результаты не
перезаписываются; dump не публикуется и не содержит рабочих данных.
## Вопросы защиты и ограничения
1. **Что подтверждает COMMIT?** Транзакция завершена и видна следующим
запросам согласно isolation. Проверены fsync=on, synchronous_commit=on,
full_page_writes=on. WAL flush защищает штатный durability сценарий,
но не потерю единственного диска/VM/host и не заменяет backup.
2. **Где сессии и что даст flush cache?** Сессии в PG, в cache только User.
Flush/restart cache даёт miss, но токен остаётся действующим; logout
действует через PG независимо от TTL карточки.
3. **Auth против ACL?** Сессия определяет пользователя. Отдельная проверка
membership определяет доступ к чату; посторонний получает такой же 404,
как отсутствующий ресурс. Успешный login сам по себе доступ не даёт.
4. **KDF и checksum?** Человеческий пароль имеет ограниченную энтропию,
поэтому нужен дорогой Argon2id с солью. Случайный 256-битный токен можно
индексировать быстрым SHA-256 digest; checksum backup решает проверку
целостности данных, а не защиту пароля.
Один PG/cache/API на host — не HA. Нет TLS на внешнем маршруте, стенд
только loopback и синтетические данные. PG init-role владеет учебной БД
и имеет широкие права; production разделение SQL-ролей не заявляется.
Backup snapshot не сохраняет последующие сообщения/отзывы сессий; без
WAL-архива и recovery policy это неизбежное ограничение восстановления.
POST сообщения ЛР2 не идемпотентен: после неизвестного исхода COMMIT
автоматический retry может создать дубль; это задача ЛР3.
## Источники и помощь
README/COURSE/contracts/tests преподавателя; справочные методические наборы;
[PostgreSQL](https://www.postgresql.org/docs/18/),
[Psycopg](https://www.psycopg.org/psycopg3/docs/),
[Argon2](https://argon2-cffi.readthedocs.io/),
[Valkey](https://valkey.io/topics/), FastAPI/Pydantic/Uvicorn.
Поведение client timeout дополнительно сверено с установленным кодом
psycopg3.3.6 и реальной паузой PostgreSQL.
Помощь ИИ использована при разборе требований, реализации, тестах и отчёте;
измерения подтверждены сохранёнными запусками.
+89
View File
@@ -0,0 +1,89 @@
name: mtusi-lab2-maslov
services:
postgres:
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
PGDATA: /var/lib/postgresql/data/pgdata
secrets: [postgres_password]
volumes:
- postgres_data:/var/lib/postgresql
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
cpus: 1
mem_limit: 512m
memswap_limit: 512m
cache:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/valkey.conf]
secrets:
- cache_password
- source: valkey_config
target: valkey.conf
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping']
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
memswap_limit: 128m
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab2}
pull_policy: never
command: [python, -m, src.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres: {condition: service_healthy}
cpus: 0.5
mem_limit: 256m
read_only: true
api:
image: ${API_IMAGE:-mtusi-messenger:lab2}
build:
context: .
args:
VCS_REF: ${VCS_REF:-unknown}
ports:
- "127.0.0.1:${PUBLIC_PORT:-8082}:8080"
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
secrets: [database_url, cache_url]
depends_on:
migrate: {condition: service_completed_successfully}
cache: {condition: service_healthy}
cpus: 1
mem_limit: 512m
memswap_limit: 512m
pids_limit: 128
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
stop_grace_period: 10s
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_password:
file: ./secrets/cache_password
cache_url:
file: ./secrets/cache_url
valkey_config:
file: ./secrets/valkey.conf
name: valkey.conf
volumes:
postgres_data:
+12
View File
@@ -0,0 +1,12 @@
Source commit: 7466a09
Status: INCOMPLETE, not a successful final clean-clone verification.
The first isolated clone used PUBLIC_PORT=0 (automatic host port).
Bootstrap/build/static checks, smoke and HTTP tests completed before the
public persistence stage. After restart the public client kept its original
BASE_URL and could not recover within 90 seconds. Temporary project and
its volumes were removed by the cleanup handler.
Diagnostic reproduction checks whether Docker reassigns an automatic host
port on restart. Final verification selects an available numeric host port
once, so its BASE_URL remains valid through the restart hook.
+117
View File
@@ -0,0 +1,117 @@
{
"date": "2026-11-07T05:47:11.012617+00:00",
"source_commit": "5e7c93a0362e667551c8807a674fe04927b20e53",
"fresh_secrets_environment_and_volumes": true,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": 51761,
"static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\nuv run --locked ruff format src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\n27 files left unchanged\nuv run --locked ruff check src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\nAll checks passed!",
"internal_store_ports_not_published": true,
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:51761\" \nPASS: lab2 public API scenario; infrastructure and grading evidence remain separate.",
"http_tests": "........................................................................ [ 90%]\n........ [100%]\n80 passed in 93.23s (0:01:33)",
"public_persistence": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"http://127.0.0.1:51761\" \nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.",
"verify_lifecycle": {
"date": "2026-11-07T05:49:09.842951+00:00",
"commit": "5e7c93a0362e667551c8807a674fe04927b20e53",
"command": "python -m scripts.verify_lifecycle",
"runtime_source_hashes_match": true,
"image_revision": "5e7c93a0362e667551c8807a674fe04927b20e53",
"persistence": {
"absolute_expiry_unchanged": true,
"same_token": true,
"same_message": true,
"before": {
"api": {
"container": "52efca283e03",
"pid": 32752,
"started_at": "2026-11-07T05:49:05.222283053Z"
},
"postgres": {
"container": "4a8a9b88ac4b",
"pid": 32547,
"started_at": "2026-11-07T05:49:04.838047886Z"
},
"cache": {
"container": "590272a808de",
"pid": 32569,
"started_at": "2026-11-07T05:49:04.863209469Z"
}
},
"after": {
"api": {
"container": "52efca283e03",
"pid": 33639,
"started_at": "2026-11-07T05:49:11.700302333Z"
},
"postgres": {
"container": "4a8a9b88ac4b",
"pid": 33460,
"started_at": "2026-11-07T05:49:11.381633375Z"
},
"cache": {
"container": "590272a808de",
"pid": 33501,
"started_at": "2026-11-07T05:49:11.40642775Z"
}
},
"seconds": 5.98190225000144
},
"migration_repeat": "PASS",
"bootstrap_preserves_secrets": true,
"credential_storage": {
"argon2id": true,
"digest_bytes": 32,
"raw_password_and_token_absent_in_dump": true
},
"cache_hits": 20,
"cache_stopped": {
"status": 200,
"seconds": 0.07452341700263787
},
"cache_flush_preserves_truth": true,
"postgres_pause": {
"cache_card_present_before_failure": true,
"status": 503,
"seconds": 4.043605250000837,
"cached_identity_not_trusted": true
},
"postgres_stop": {
"cache_card_present_before_failure": true,
"status": 503,
"seconds": 0.01112437499978114,
"cached_identity_not_trusted": true
},
"chat_transaction_rollback": {
"injected_failure_status": 500,
"remaining_chat_rows": 0
},
"expiry": {
"ttl_seconds": 2,
"status_after_expiry": 401
},
"logout_after_cache_hit": "PASS",
"result": "PASS"
},
"verify_pool": {
"date": "2026-11-07T05:49:46.535430+00:00",
"commit": "5e7c93a0362e667551c8807a674fe04927b20e53",
"command": "python -m scripts.verify_pool",
"pool_size": 1,
"blocked_insert_observed_in_pg_stat_activity": true,
"pool_wait_seconds": 1.005813499999931,
"pool_timeout_status": 503,
"statement_timeout_status": 503,
"failed_write_rolled_back": true,
"write_after_recovery": 201,
"result": "PASS"
},
"applied_migration_checksum_guard": "PASS",
"research_repeated": {
"result": "PASS",
"messages": 101110,
"plans": 90,
"restored_snapshot_equal": true
},
"fsck": "",
"checks_completed": true,
"result": "PASS"
}
+117
View File
@@ -0,0 +1,117 @@
{
"date": "2026-11-07T05:26:38.162144+00:00",
"source_commit": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
"fresh_secrets_environment_and_volumes": true,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": 49287,
"static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n<home>/.local/bin/uv run --locked ruff format src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\n26 files left unchanged\n<home>/.local/bin/uv run --locked ruff check src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\nAll checks passed!",
"internal_store_ports_not_published": true,
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:49287\" \nPASS: lab2 public API scenario; infrastructure and grading evidence remain separate.",
"http_tests": "........................................................................ [ 93%]\n..... [100%]\n77 passed in 86.00s (0:01:25)",
"public_persistence": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"http://127.0.0.1:49287\" \nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.",
"verify_lifecycle": {
"date": "2026-11-07T05:28:29.159541+00:00",
"commit": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
"command": "python -m scripts.verify_lifecycle",
"runtime_source_hashes_match": true,
"image_revision": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
"persistence": {
"absolute_expiry_unchanged": true,
"same_token": true,
"same_message": true,
"before": {
"api": {
"container": "4cc53c6190ac",
"pid": 89123,
"started_at": "2026-11-07T05:28:24.620912381Z"
},
"postgres": {
"container": "910b02e40597",
"pid": 88965,
"started_at": "2026-11-07T05:28:24.260037964Z"
},
"cache": {
"container": "d47529714a4a",
"pid": 88983,
"started_at": "2026-11-07T05:28:24.279300006Z"
}
},
"after": {
"api": {
"container": "4cc53c6190ac",
"pid": 89990,
"started_at": "2026-11-07T05:28:30.971012968Z"
},
"postgres": {
"container": "910b02e40597",
"pid": 89812,
"started_at": "2026-11-07T05:28:30.640581301Z"
},
"cache": {
"container": "d47529714a4a",
"pid": 89865,
"started_at": "2026-11-07T05:28:30.700865926Z"
}
},
"seconds": 5.304535000002943
},
"migration_repeat": "PASS",
"bootstrap_preserves_secrets": true,
"credential_storage": {
"argon2id": true,
"digest_bytes": 32,
"raw_password_and_token_absent_in_dump": true
},
"cache_hits": 20,
"cache_stopped": {
"status": 200,
"seconds": 0.013735125001403503
},
"cache_flush_preserves_truth": true,
"postgres_pause": {
"cache_card_present_before_failure": true,
"status": 503,
"seconds": 4.019069417001447,
"cached_identity_not_trusted": true
},
"postgres_stop": {
"cache_card_present_before_failure": true,
"status": 503,
"seconds": 0.005429041993920691,
"cached_identity_not_trusted": true
},
"chat_transaction_rollback": {
"injected_failure_status": 500,
"remaining_chat_rows": 0
},
"expiry": {
"ttl_seconds": 2,
"status_after_expiry": 401
},
"logout_after_cache_hit": "PASS",
"result": "PASS"
},
"verify_pool": {
"date": "2026-11-07T05:29:04.541663+00:00",
"commit": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
"command": "python -m scripts.verify_pool",
"pool_size": 1,
"blocked_insert_observed_in_pg_stat_activity": true,
"pool_wait_seconds": 1.0053121249948163,
"pool_timeout_status": 503,
"statement_timeout_status": 503,
"failed_write_rolled_back": true,
"write_after_recovery": 201,
"result": "PASS"
},
"applied_migration_checksum_guard": "PASS",
"research_repeated": {
"result": "PASS",
"messages": 101110,
"plans": 90,
"restored_snapshot_equal": true
},
"fsck": "",
"checks_completed": true,
"result": "PASS"
}
+8
View File
@@ -0,0 +1,8 @@
{
"date": "2026-11-07",
"source_commit": "2f0a867",
"public_smoke": "PASS: lab2 public API scenario",
"student_tests": {"passed": 77, "seconds": 91.82},
"command": "NO_PROXY=localhost,127.0.0.1 make test; pytest student-tests -q -x",
"note": "HTTP checks against real PostgreSQL/Valkey/API Compose stack; credentials remain in memory."
}
+79
View File
@@ -0,0 +1,79 @@
{
"date": "2026-11-07T05:05:22.285580+00:00",
"commit": "5fb1f19de15552321d2c9f5449e088e6452a7189",
"command": "python -m scripts.verify_lifecycle",
"runtime_source_hashes_match": true,
"image_revision": "2f0a8672be67af9e62d2693c3df9bd683fcdae57",
"persistence": {
"same_token": true,
"same_message": true,
"before": {
"api": {
"container": "2b25734b0954",
"pid": 1409,
"started_at": "2026-11-07T05:00:09.733545264Z"
},
"postgres": {
"container": "29e277263007",
"pid": 1109,
"started_at": "2026-11-07T05:00:04.997609011Z"
},
"cache": {
"container": "9f6a93f4f941",
"pid": 1041,
"started_at": "2026-11-07T05:00:04.881758678Z"
}
},
"after": {
"api": {
"container": "2b25734b0954",
"pid": 21469,
"started_at": "2026-11-07T05:05:24.139344714Z"
},
"postgres": {
"container": "29e277263007",
"pid": 21288,
"started_at": "2026-11-07T05:05:23.819680089Z"
},
"cache": {
"container": "9f6a93f4f941",
"pid": 21329,
"started_at": "2026-11-07T05:05:23.858910756Z"
}
},
"seconds": 5.230442958010826
},
"migration_repeat": "PASS",
"bootstrap_preserves_secrets": true,
"credential_storage": {
"argon2id": true,
"digest_bytes": 32,
"raw_password_and_token_absent_in_dump": true
},
"cache_hits": 20,
"cache_stopped": {
"status": 200,
"seconds": 0.11006329099473078
},
"cache_flush_preserves_truth": true,
"postgres_pause": {
"status": 503,
"seconds": 4.011241999993217,
"cached_identity_not_trusted": true
},
"postgres_stop": {
"status": 503,
"seconds": 0.003582208009902388,
"cached_identity_not_trusted": true
},
"chat_transaction_rollback": {
"injected_failure_status": 500,
"remaining_chat_rows": 0
},
"expiry": {
"ttl_seconds": 2,
"status_after_expiry": 401
},
"logout_after_cache_hit": "PASS",
"result": "PASS"
}
+13
View File
@@ -0,0 +1,13 @@
{
"date": "2026-11-07T05:15:33.380498+00:00",
"commit": "171b6de9c87c14cdfb1f9ec6a58aab7101c23d6a",
"command": "python -m scripts.verify_pool",
"pool_size": 1,
"blocked_insert_observed_in_pg_stat_activity": true,
"pool_wait_seconds": 1.0045806250127498,
"pool_timeout_status": 503,
"statement_timeout_status": 503,
"failed_write_rolled_back": true,
"write_after_recovery": 201,
"result": "PASS"
}
+10
View File
@@ -0,0 +1,10 @@
{
"date": "2026-11-07",
"scope": "own temporary API, same container before and after docker restart",
"requested_host_port": "automatic",
"before": 65424,
"after": 65441,
"port_changed": true,
"result": "CONFIRMED",
"fix": "Choose a free numeric host port once for the clean-clone Compose project. Keep BASE_URL stable through the public persistence hook."
}
+10
View File
@@ -0,0 +1,10 @@
{
"result": "EXPECTED_FAILURE_BEFORE_FIX",
"checks": {
"utf16_json": "201, expected 400",
"utf32_json": "201, expected 400",
"leading_zero_limit": "400, expected 200"
},
"command": "BASE_URL=<old-image-url> .venv/bin/python -m pytest student-tests/test_contract_edges.py -q",
"old_revision": "2f0a8672be67af9e62d2693c3df9bd683fcdae57"
}
+8
View File
@@ -0,0 +1,8 @@
Date: 2026-11-07
Runtime source: 2f0a8672be67af9e62d2693c3df9bd683fcdae57
Command: make test-persistence ACTION_SCRIPT=scripts/restart.sh
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
Actual API/PostgreSQL/Valkey restart evidence: lifecycle.json
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

+31
View File
@@ -0,0 +1,31 @@
messages,position,variant,repeats,median_sql_ms,min_sql_ms,max_sql_ms,median_buffers
10,head,chat_only,3,0.011,0.011,0.015,3
10,head,keyset,3,0.006,0.006,0.008,4
10,middle,chat_only,3,0.01,0.008,0.013,3
10,middle,keyset,3,0.005,0.005,0.007,4
10,tail,chat_only,3,0.011,0.01,0.011,3
10,tail,keyset,3,0.006,0.006,0.007,4
100,head,chat_only,3,0.038,0.034,0.039,6
100,head,keyset,3,0.016,0.016,0.018,7
100,middle,chat_only,3,0.03,0.026,0.034,6
100,middle,keyset,3,0.015,0.014,0.015,5
100,tail,chat_only,3,0.028,0.027,0.031,6
100,tail,keyset,3,0.016,0.015,0.016,6
1000,head,chat_only,3,0.197,0.195,0.205,25
1000,head,keyset,3,0.018,0.016,0.02,5
1000,middle,chat_only,3,0.152,0.136,0.158,25
1000,middle,keyset,3,0.026,0.017,0.032,6
1000,tail,chat_only,3,0.092,0.08,0.106,25
1000,tail,keyset,3,0.016,0.015,0.019,6
10000,head,chat_only,3,1.801,1.755,1.845,224
10000,head,keyset,3,0.019,0.018,0.019,6
10000,middle,chat_only,3,1.187,1.17,1.198,224
10000,middle,keyset,3,0.016,0.016,0.018,5
10000,tail,chat_only,3,0.624,0.603,0.878,224
10000,tail,keyset,3,0.016,0.016,0.017,6
90000,head,chat_only,3,16.501,9.887,22.44,2209
90000,head,keyset,3,0.02,0.019,0.021,5
90000,middle,chat_only,3,7.237,7.082,8.505,2209
90000,middle,keyset,3,0.017,0.016,0.018,5
90000,tail,chat_only,3,4.287,4.282,4.674,2194
90000,tail,keyset,3,0.016,0.016,0.018,5
1 messages position variant repeats median_sql_ms min_sql_ms max_sql_ms median_buffers
2 10 head chat_only 3 0.011 0.011 0.015 3
3 10 head keyset 3 0.006 0.006 0.008 4
4 10 middle chat_only 3 0.01 0.008 0.013 3
5 10 middle keyset 3 0.005 0.005 0.007 4
6 10 tail chat_only 3 0.011 0.01 0.011 3
7 10 tail keyset 3 0.006 0.006 0.007 4
8 100 head chat_only 3 0.038 0.034 0.039 6
9 100 head keyset 3 0.016 0.016 0.018 7
10 100 middle chat_only 3 0.03 0.026 0.034 6
11 100 middle keyset 3 0.015 0.014 0.015 5
12 100 tail chat_only 3 0.028 0.027 0.031 6
13 100 tail keyset 3 0.016 0.015 0.016 6
14 1000 head chat_only 3 0.197 0.195 0.205 25
15 1000 head keyset 3 0.018 0.016 0.02 5
16 1000 middle chat_only 3 0.152 0.136 0.158 25
17 1000 middle keyset 3 0.026 0.017 0.032 6
18 1000 tail chat_only 3 0.092 0.08 0.106 25
19 1000 tail keyset 3 0.016 0.015 0.019 6
20 10000 head chat_only 3 1.801 1.755 1.845 224
21 10000 head keyset 3 0.019 0.018 0.019 6
22 10000 middle chat_only 3 1.187 1.17 1.198 224
23 10000 middle keyset 3 0.016 0.016 0.018 5
24 10000 tail chat_only 3 0.624 0.603 0.878 224
25 10000 tail keyset 3 0.016 0.016 0.017 6
26 90000 head chat_only 3 16.501 9.887 22.44 2209
27 90000 head keyset 3 0.02 0.019 0.021 5
28 90000 middle chat_only 3 7.237 7.082 8.505 2209
29 90000 middle keyset 3 0.017 0.016 0.018 5
30 90000 tail chat_only 3 4.287 4.282 4.674 2194
31 90000 tail keyset 3 0.016 0.016 0.018 5
+20
View File
@@ -0,0 +1,20 @@
# Планы запросов и восстановление
Три повтора, 101110 сообщений в 5 неравномерных чатах.
| Индекс | SQL середины чата90k, мс | Буферы | Размер, bytes | INSERT+COMMIT1000, мс | WAL, bytes |
|---|---:|---:|---:|---:|---:|
| chat_only | 7.237 | 2209 | 729088 | 37.53 | 347536 |
| keyset | 0.017 | 5 | 5980160 | 11.55 | 427672 |
Dump: 0.480 s, 3026259 bytes.
Restore в новую БД: 0.664 s.
Проверены count всех таблиц, SHA-256 всей истории и 15 полных сообщений.
Одна отдельная SQL-запись после snapshot не восстановлена — ожидаемая
граница snapshot backup. Это не HTTP RPO и не гарантия HA.
Время EXPLAIN — SQL execution time, не HTTP latency. Цена записи включает
COMMIT 1000 строк; WAL — глобальный счётчик PG и может включать фон.
Индекс keyset больше и даёт больше WAL, но в этом опыте INSERT быстрее:
гипотеза роста именно времени записи не подтвердилась. Причина механизма
такого различия отдельно не профилировалась.
+13
View File
@@ -0,0 +1,13 @@
# Область первых проверок
В lifecycle.json подтверждены restart всех трёх процессов, тот же токен,
expiry/logout, cache hits и bounded 503 при остановке/паузе PostgreSQL.
Однако наличие карточки в кэше непосредственно перед отказом PG отдельно
не проверялось: после рестарта cache могло ещё действовать окно backoff.
Поэтому cached_identity_not_trusted в этой первой записи не является
самостоятельным доказательством сохранённого cache hit при отказе PG.
Финальный скрипт усилен: ждёт окончания backoff, проверяет EXISTS карточки
перед каждым воздействием на PG и сравнивает абсолютный expires_at до/после
restart. Этот вариант успешно повторён в финальном чистом clone; результат
в clean-clone.json. Runtime API при этом не менялся.
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+31
View File
@@ -0,0 +1,31 @@
[project]
name = "mtusi-messenger-lab2"
version = "1.0.0"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi==0.143.3", "pydantic==2.14.1", "uvicorn==0.54.0",
"psycopg[binary,pool]==3.3.6", "argon2-cffi>=25,<26", "redis==8.1.0",
]
[dependency-groups]
dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validator==0.7.2", "psutil>=7,<8", "matplotlib>=3.10,<4"]
[tool.ruff]
line-length = 79
target-version = "py314"
include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py", "scripts/verify_*.py"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
[tool.ruff.lint.per-file-ignores]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
View File
+134
View File
@@ -0,0 +1,134 @@
import argparse
import csv
import json
import statistics
from collections import defaultdict
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
def main():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=Path)
args = parser.parse_args()
result = json.loads(args.input.read_text())
groups = defaultdict(list)
for row in result["reads"]:
groups[(row["chat_messages"], row["position"], row["variant"])].append(
row
)
rows = []
for (size, position, variant), values in sorted(groups.items()):
ms = [v["execution_ms"] for v in values]
rows.append(
{
"messages": size,
"position": position,
"variant": variant,
"repeats": len(values),
"median_sql_ms": statistics.median(ms),
"min_sql_ms": min(ms),
"max_sql_ms": max(ms),
"median_buffers": statistics.median(
v["shared_hits"] + v["shared_reads"] for v in values
),
}
)
directory = args.input.parent
with (directory / "query-summary.csv").open("w") as stream:
writer = csv.DictWriter(
stream, fieldnames=list(rows[0]), lineterminator="\n"
)
writer.writeheader()
writer.writerows(rows)
lines = [
"# Планы запросов и восстановление",
"",
"Три повтора, 101110 сообщений в 5 неравномерных чатах.",
"",
"| Индекс | SQL середины чата90k, мс | Буферы | Размер, bytes | INSERT+COMMIT1000, мс | WAL, bytes |",
"|---|---:|---:|---:|---:|---:|",
]
for variant in ["chat_only", "keyset"]:
read = next(
r
for r in rows
if r["messages"] == 90000
and r["position"] == "middle"
and r["variant"] == variant
)
writes = [r for r in result["writes"] if r["variant"] == variant]
index_bytes = statistics.median(
r["bytes"] for r in result["indexes"] if r["variant"] == variant
)
latency = statistics.median(
r["insert_and_commit_seconds"] * 1000 for r in writes
)
wal = statistics.median(r["wal_bytes"] for r in writes)
lines.append(
f"| {variant} | {read['median_sql_ms']:.3f} | {read['median_buffers']:.0f} | {index_bytes:.0f} | {latency:.2f} | {wal:.0f} |"
)
lines.extend(
[
"",
f"Dump: {result['backup_seconds']:.3f} s, {result['backup_bytes']} bytes.",
f"Restore в новую БД: {result['restore_seconds']:.3f} s.",
"Проверены count всех таблиц, SHA-256 всей истории и 15 полных сообщений.",
"Одна отдельная SQL-запись после snapshot не восстановлена — ожидаемая",
"граница snapshot backup. Это не HTTP RPO и не гарантия HA.",
"",
"Время EXPLAIN — SQL execution time, не HTTP latency. Цена записи включает",
"COMMIT 1000 строк; WAL — глобальный счётчик PG и может включать фон.",
"Индекс keyset больше и даёт больше WAL, но в этом опыте INSERT быстрее:",
"гипотеза роста именно времени записи не подтвердилась. Причина механизма",
"такого различия отдельно не профилировалась.",
]
)
(directory / "query-summary.md").write_text("\n".join(lines) + "\n")
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
for variant in ["chat_only", "keyset"]:
selected = sorted(
(
r
for r in rows
if r["variant"] == variant and r["position"] == "middle"
),
key=lambda r: r["messages"],
)
sizes = [r["messages"] for r in selected]
axes[0].plot(
sizes,
[r["median_sql_ms"] for r in selected],
marker="o",
label=variant,
)
axes[1].plot(
sizes,
[r["median_buffers"] for r in selected],
marker="o",
label=variant,
)
for ax, ylabel in zip(
axes, ["SQL execution time, ms", "Shared buffers touched"], strict=True
):
ax.set_xscale("log")
ax.set_yscale("log")
ax.set_xlabel("Messages in chat")
ax.set_ylabel(ylabel)
ax.grid(alpha=0.3)
ax.legend()
fig.suptitle(
"PostgreSQL 1 CPU / 512 MiB; midpoint keyset page; median of 3"
)
fig.tight_layout()
fig.savefig(directory / "query-plans.png", dpi=150)
plt.close(fig)
print("\n".join(lines))
if __name__ == "__main__":
main()
+442
View File
@@ -0,0 +1,442 @@
"""Изолированная БД: планы, цена записи, backup/restore без рабочих данных."""
import argparse
import hashlib
import json
import resource
import time
from datetime import UTC, datetime, timedelta
from pathlib import Path
from uuid import UUID, uuid4, uuid5
import psycopg
from psycopg import sql
from psycopg.conninfo import make_conninfo
from src.settings import secret
NAMESPACE = UUID("2e52c91b-b28f-4990-988c-7e8e15c9f131")
USER = uuid5(NAMESPACE, "research-user")
ORIGIN = datetime(2026, 1, 1, tzinfo=UTC)
SIZES = [90_000, 10_000, 1000, 100, 10]
def connect(database):
return psycopg.connect(
make_conninfo(secret("DATABASE_URL"), dbname=database),
autocommit=True,
connect_timeout=3,
prepare_threshold=None,
options="-c statement_timeout=60000 -c timezone=UTC",
)
def safe_database(name):
import re
if not re.fullmatch(r"(?:research|restored)_[a-f0-9]{12}", name):
raise ValueError("Refusing database outside research scope")
def snapshot(conn):
counts = {}
for table in [
"users",
"chats",
"members",
"messages",
"sessions",
"schema_migrations",
]:
counts[table] = conn.execute(
sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table))
).fetchone()[0]
checksum = hashlib.sha256()
with conn.transaction(), conn.cursor(name="snapshot_messages") as cursor:
cursor.execute(
"SELECT id,chat_id,sender_id,text,created_at "
"FROM messages ORDER BY id"
)
for row in cursor:
checksum.update(
json.dumps(
[str(value) for value in row], separators=(",", ":")
).encode()
)
checksum.update(b"\n")
selected = []
for index, size in enumerate(SIZES):
chat = uuid5(NAMESPACE, "chat:" + str(index))
for offset in [0, size // 2, size - 1]:
row = conn.execute(
"SELECT id,chat_id,sender_id,text,created_at "
"FROM messages WHERE chat_id=%s "
"ORDER BY created_at,id OFFSET %s LIMIT 1",
(chat, offset),
).fetchone()
selected.append([str(value) for value in row])
return {
"counts": counts,
"messages_sha256": checksum.hexdigest(),
"selected_messages": selected,
}
def inside_plan(database):
safe_database(database)
with connect("messenger") as admin:
admin.execute(
sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database))
)
with connect(database) as conn:
migration = Path("migrations/001_core.sql").read_bytes()
conn.execute(migration.decode())
conn.execute(
"CREATE TABLE schema_migrations (name text PRIMARY KEY, "
"checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute(
"INSERT INTO schema_migrations(name,checksum) VALUES(%s,%s)",
("001_core.sql", hashlib.sha256(migration).hexdigest()),
)
conn.execute(
"INSERT INTO users VALUES(%s,%s,%s,%s,%s)",
(USER, "research_user", "Synthetic", "not-a-login", ORIGIN),
)
cursors = []
for index, size in enumerate(SIZES):
chat = uuid5(NAMESPACE, "chat:" + str(index))
conn.execute(
"INSERT INTO chats VALUES(%s,%s,%s)",
(chat, "Research " + str(index), ORIGIN),
)
conn.execute("INSERT INTO members VALUES(%s,%s,0)", (chat, USER))
conn.execute(
"INSERT INTO messages SELECT "
"md5(%s || ':' || n::text)::uuid,%s,%s,"
"rpad('message ' || n,80,'x'),%s + n*interval '1 microsecond' "
"FROM generate_series(1,%s) n",
(str(chat), chat, USER, ORIGIN, size),
)
for position, offset in [
("head", 0),
("middle", size // 2),
("tail", max(0, size - 52)),
]:
row = conn.execute(
"SELECT created_at,id FROM messages WHERE chat_id=%s "
"ORDER BY created_at,id OFFSET %s LIMIT 1",
(chat, offset),
).fetchone()
cursors.append((size, position, chat, row[0], row[1]))
assert (
conn.execute("SELECT count(*) FROM messages").fetchone()[0]
>= 100_000
)
query = (
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
"WHERE chat_id=%s AND (created_at,id)>(%s,%s) "
"ORDER BY created_at,id LIMIT 51"
)
reads, writes, indexes = [], [], []
for repeat, order in enumerate(
[
["chat_only", "keyset"],
["keyset", "chat_only"],
["chat_only", "keyset"],
],
1,
):
for variant in order:
conn.execute("DROP INDEX IF EXISTS messages_history_idx")
conn.execute("DROP INDEX IF EXISTS messages_chat_only_idx")
if variant == "chat_only":
conn.execute(
"CREATE INDEX messages_chat_only_idx "
"ON messages(chat_id)"
)
index_name = "messages_chat_only_idx"
else:
conn.execute(
"CREATE INDEX messages_history_idx "
"ON messages(chat_id,created_at,id)"
)
index_name = "messages_history_idx"
conn.execute("VACUUM (ANALYZE) messages")
index_bytes = conn.execute(
"SELECT pg_relation_size(%s)", (index_name,)
).fetchone()[0]
indexes.append(
{
"repeat": repeat,
"variant": variant,
"bytes": index_bytes,
}
)
for size, position, chat, at, mid in cursors:
parameters = (chat, at, mid)
conn.execute(query, parameters).fetchall()
plan = conn.execute(
"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query,
parameters,
).fetchone()[0][0]
reads.append(
{
"repeat": repeat,
"variant": variant,
"chat_messages": size,
"position": position,
"execution_ms": plan["Execution Time"],
"planning_ms": plan["Planning Time"],
"shared_hits": plan["Plan"].get(
"Shared Hit Blocks", 0
),
"shared_reads": plan["Plan"].get(
"Shared Read Blocks", 0
),
"plan": plan,
}
)
# Окно записи включает COMMIT; cleanup за границей окна.
batch = "write:" + uuid4().hex
chat = uuid5(NAMESPACE, "chat:0")
lsn_before = conn.execute(
"SELECT pg_current_wal_insert_lsn()"
).fetchone()[0]
started = time.perf_counter()
with conn.transaction():
conn.execute(
"INSERT INTO messages SELECT "
"md5(%s || ':' || n::text)::uuid,%s,%s,"
"rpad('write ' || n,80,'x'),%s + "
"n*interval '1 microsecond' "
"FROM generate_series(1,1000) n",
(batch, chat, USER, ORIGIN + timedelta(days=1)),
)
elapsed = time.perf_counter() - started
wal = conn.execute(
"SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),%s)",
(lsn_before,),
).fetchone()[0]
writes.append(
{
"repeat": repeat,
"variant": variant,
"inserted": 1000,
"insert_and_commit_seconds": elapsed,
"wal_bytes": int(wal),
}
)
conn.execute(
"DELETE FROM messages WHERE id IN "
"(SELECT md5(%s || ':' || n::text)::uuid "
"FROM generate_series(1,1000) n)",
(batch,),
)
assert conn.execute(
"SELECT count(*) FROM messages"
).fetchone()[0] == sum(SIZES)
result = {
"dataset": {
"chat_sizes": SIZES,
"messages": sum(SIZES),
"text_characters": 80,
"source": (
"synthetic SQL seed; no production API "
"or working tables modified"
),
},
"reads": reads,
"writes": writes,
"indexes": indexes,
"snapshot_before_backup": snapshot(conn),
"research_worker_maxrss_kib": resource.getrusage(
resource.RUSAGE_SELF
).ru_maxrss,
}
return result
def inside_after(database):
safe_database(database)
with connect(database) as conn:
value = uuid4()
conn.execute(
"INSERT INTO messages VALUES(%s,%s,%s,%s,%s)",
(
value,
uuid5(NAMESPACE, "chat:0"),
USER,
"Committed after backup snapshot",
ORIGIN + timedelta(days=2),
),
)
return {
"post_backup_message": str(value),
"count": conn.execute("SELECT count(*) FROM messages").fetchone()[
0
],
}
def inside_snapshot(database):
safe_database(database)
with connect(database) as conn:
return snapshot(conn)
def inside_cleanup(databases):
with connect("messenger") as admin:
for database in databases:
safe_database(database)
admin.execute(
sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
sql.Identifier(database)
)
)
return {"scoped_databases_removed": databases}
def outside(output):
from scripts.stand import ROOT, command, compose
if output.exists():
raise SystemExit("Choose a new output file")
identifier = uuid4().hex[:12]
database, restored = "research_" + identifier, "restored_" + identifier
dump_path = "/tmp/" + database + ".dump"
mount = str(ROOT / "research") + ":/app/research:ro"
def inside(phase, selected):
return json.loads(
compose(
"run",
"--rm",
"--no-deps",
"-v",
mount,
"api",
"python",
"-m",
"research.query_plan",
"--phase",
phase,
"--database",
selected,
)
)
result = {
"date": datetime.now(UTC).isoformat(),
"commit": command(["git", "rev-parse", "HEAD"]),
"command": "python -m research.query_plan",
"db_cpu_limit": 1,
"db_memory_limit_bytes": 536870912,
"prepared_statements": "disabled for comparable custom EXPLAIN plans",
}
try:
result.update(inside("plan", database))
started = time.perf_counter()
compose(
"exec",
"-T",
"postgres",
"pg_dump",
"-U",
"messenger",
"-Fc",
"-d",
database,
"-f",
dump_path,
)
result["backup_seconds"] = time.perf_counter() - started
result["backup_bytes"] = int(
compose("exec", "-T", "postgres", "stat", "-c", "%s", dump_path)
)
after = inside("after", database)
started = time.perf_counter()
compose(
"exec", "-T", "postgres", "createdb", "-U", "messenger", restored
)
compose(
"exec",
"-T",
"postgres",
"pg_restore",
"-U",
"messenger",
"--exit-on-error",
"-d",
restored,
dump_path,
)
result["restore_seconds"] = time.perf_counter() - started
checked = inside("snapshot", restored)
expected = result["snapshot_before_backup"]
assert expected == checked
assert after["count"] == expected["counts"]["messages"] + 1
result["restore_verification"] = {
"snapshot_equal": True,
"selected_messages_checked": len(checked["selected_messages"]),
"restored": checked,
"post_snapshot_commits_not_restored": 1,
"source_count_after_backup": after["count"],
"post_snapshot_message": after["post_backup_message"],
}
result["postgres_cgroup_memory_current"] = int(
compose(
"exec",
"-T",
"postgres",
"cat",
"/sys/fs/cgroup/memory.current",
)
)
result["postgres_cgroup_memory_peak"] = int(
compose(
"exec", "-T", "postgres", "cat", "/sys/fs/cgroup/memory.peak"
)
)
result["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS:",
result["dataset"]["messages"],
"messages;",
len(result["reads"]),
"EXPLAIN plans; backup/restore verified",
)
finally:
# Удаляются только две БД с UUID именно этого запуска.
for selected in [database, restored]:
inside("cleanup", selected)
compose("exec", "-T", "postgres", "rm", "-f", dump_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", type=Path)
parser.add_argument(
"--phase", choices=["plan", "after", "snapshot", "cleanup"]
)
parser.add_argument("--database")
args = parser.parse_args()
if args.phase:
function = {
"plan": inside_plan,
"after": inside_after,
"snapshot": inside_snapshot,
"cleanup": lambda name: inside_cleanup([name]),
}[args.phase]
print(json.dumps(function(args.database), ensure_ascii=False))
else:
outside(args.output)
if __name__ == "__main__":
main()
+47
View File
@@ -0,0 +1,47 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose restart api postgres cache >/dev/null
docker compose up -d --no-recreate --wait >/dev/null
+137
View File
@@ -0,0 +1,137 @@
import json
import os
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8082")
def command(args, *, input=None):
result = subprocess.run(
args, cwd=ROOT, input=input, capture_output=True, text=True
)
if result.returncode:
raise RuntimeError(
"Command failed: "
+ args[0]
+ "; "
+ result.stderr.replace(str(ROOT), "<repository>")
)
return result.stdout.strip()
def compose(*args):
return command(["docker", "compose", *args])
def sql(query, *, database="messenger"):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
database,
"-A",
"-t",
"-v",
"ON_ERROR_STOP=1",
"-c",
query,
)
def cache(command_name, *args):
script = (
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
'exec valkey-cli "$@"'
)
return compose(
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
)
def ready(base=BASE, *, timeout=30):
end = time.monotonic() + timeout
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
while time.monotonic() < end:
try:
if client.get("/health/ready").status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.2)
raise TimeoutError("API readiness")
def actor(client):
username = "verify_" + uuid4().hex[:20]
password = "Synthetic-" + uuid4().hex + ":🌍"
result = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert result.status_code == 201
user = result.json()
session = client.post(
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
)
assert session.status_code == 201
return user, password, session.json()
def auth(session):
return {"Authorization": "Bearer " + session["token"]}
def extra_api(*, ttl=3600, pool_size=1):
name = "mtusi-lab2-test-" + uuid4().hex[:10]
compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"-e",
"SESSION_TTL_SECONDS=" + str(ttl),
"-e",
"DB_POOL_SIZE=" + str(pool_size),
"-p",
"127.0.0.1::8080",
"api",
)
try:
info = json.loads(command(["docker", "inspect", name]))[0]
base = (
"http://127.0.0.1:"
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
)
ready(base)
return name, base
except Exception:
command(["docker", "rm", "-f", name])
raise
def instances():
data = {}
for role in ["api", "postgres", "cache"]:
cid = compose("ps", "-q", role)
info = json.loads(command(["docker", "inspect", cid]))[0]
data[role] = {
"container": cid[:12],
"pid": info["State"]["Pid"],
"started_at": info["State"]["StartedAt"],
}
return data
+247
View File
@@ -0,0 +1,247 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def run(args, *, cwd, env, expected=0):
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
def redact(text):
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
raise RuntimeError(redact(result.stdout + result.stderr))
return redact(result.stdout.strip())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab2-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab2-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab2-clean-") as temporary:
clone = Path(temporary) / "lab2"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
api = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
)
info = json.loads(
run(["docker", "inspect", api], cwd=clone, env=env)
)[0]
endpoint = info["NetworkSettings"]["Ports"]["8080/tcp"][0]
assert endpoint["HostIp"] == "127.0.0.1"
env["BASE_URL"] = "http://127.0.0.1:" + endpoint["HostPort"]
for role, port in [
("postgres", "5432/tcp"),
("cache", "6379/tcp"),
]:
cid = run(
["docker", "compose", "ps", "-q", role], cwd=clone, env=env
)
state = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert not state["NetworkSettings"]["Ports"].get(port)
receipt["internal_store_ports_not_published"] = True
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
receipt["public_persistence"] = run(
[
"make",
"test-persistence",
"ACTION_SCRIPT=scripts/restart.sh",
"BASE_URL=" + env["BASE_URL"],
],
cwd=clone,
env=env,
)
for module, filename in [
("verify_lifecycle", "clean-lifecycle.json"),
("verify_pool", "clean-pool.json"),
]:
run(
[
".venv/bin/python",
"-m",
"scripts." + module,
"--output",
"evidence/" + filename,
],
cwd=clone,
env=env,
)
receipt[module] = json.loads(
(clone / "evidence" / filename).read_text()
)
# Подменяется только временная read-only копия миграции.
altered = clone / ".work/altered-migrations"
altered.mkdir(parents=True)
migration = clone / "migrations/001_core.sql"
(altered / migration.name).write_bytes(
migration.read_bytes() + b"\n-- synthetic checksum mismatch\n"
)
attempt = subprocess.run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-v",
str(altered) + ":/app/migrations:ro",
"migrate",
],
cwd=clone,
env=env,
capture_output=True,
text=True,
)
assert attempt.returncode != 0
assert "Applied migration checksum mismatch" in attempt.stderr
receipt["applied_migration_checksum_guard"] = "PASS"
run(
[
".venv/bin/python",
"-m",
"research.query_plan",
"--output",
"evidence/clean-query-plan.json",
],
cwd=clone,
env=env,
)
research = json.loads(
(clone / "evidence/clean-query-plan.json").read_text()
)
receipt["research_repeated"] = {
"result": research["result"],
"messages": research["dataset"]["messages"],
"plans": len(research["reads"]),
"restored_snapshot_equal": research["restore_verification"][
"snapshot_equal"
],
}
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
finally:
assert project.startswith("mtusi-lab2-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(receipt, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+311
View File
@@ -0,0 +1,311 @@
import argparse
import hashlib
import json
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import (
BASE,
ROOT,
actor,
auth,
cache,
command,
compose,
extra_api,
instances,
ready,
sql,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", required=True)
args = parser.parse_args()
output = Path(args.output)
if output.exists():
raise SystemExit("Choose new output path")
receipt = {
"date": datetime.now(UTC).isoformat(),
"commit": command(["git", "rev-parse", "HEAD"]),
"command": "python -m scripts.verify_lifecycle",
}
ready()
source_hashes = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "src").glob("*.py")
}
installed_hashes = json.loads(
compose(
"exec",
"-T",
"api",
"python",
"-c",
"import hashlib,json;from pathlib import Path;"
"print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() "
"for p in Path('/app/src').glob('*.py')}))",
)
)
assert source_hashes == installed_hashes
receipt["runtime_source_hashes_match"] = True
api_info = json.loads(
command(["docker", "inspect", compose("ps", "-q", "api")])
)[0]
receipt["image_revision"] = api_info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
with httpx.Client(base_url=BASE, trust_env=False, timeout=15) as client:
user, password, session = actor(client)
headers = auth(session)
expiry_query = (
"SELECT expires_at::text FROM sessions WHERE user_id='"
+ user["id"]
+ "'"
)
expires_before = sql(expiry_query)
chat = client.post(
"/api/v1/chats",
headers=headers,
json={"title": "Lifecycle", "member_ids": [user["id"]]},
).json()
path = "/api/v1/chats/" + chat["id"] + "/messages"
message = client.post(
path, headers=headers, json={"text": "Synthetic retained message"}
)
assert message.status_code == 201
message_id = message.json()["id"]
before = instances()
started = time.perf_counter()
command(["sh", "scripts/restart.sh"])
ready()
after = instances()
assert sql(expiry_query) == expires_before
for role in before:
assert before[role]["started_at"] != after[role]["started_at"]
assert (
client.get(path, headers=headers).json()["items"][0]["id"]
== message_id
)
receipt["persistence"] = {
"absolute_expiry_unchanged": True,
"same_token": True,
"same_message": True,
"before": before,
"after": after,
"seconds": time.perf_counter() - started,
}
migration_count = sql("SELECT count(*) FROM schema_migrations")
compose("run", "--rm", "--no-deps", "migrate")
assert sql("SELECT count(*) FROM schema_migrations") == migration_count
receipt["migration_repeat"] = "PASS"
files_before = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "secrets").iterdir()
}
command(["python3", "scripts/bootstrap.py"])
assert files_before == {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "secrets").iterdir()
}
receipt["bootstrap_preserves_secrets"] = True
dump = compose(
"exec",
"-T",
"postgres",
"pg_dump",
"-U",
"messenger",
"--data-only",
"messenger",
)
assert password not in dump and session["token"] not in dump
del dump
privacy = json.loads(
sql(
"SELECT json_build_object('argon2id',"
"password_hash LIKE '$argon2id$%','digest_bytes',"
"(SELECT min(octet_length(token_digest)) FROM sessions "
"WHERE user_id=users.id)) FROM users WHERE id='"
+ user["id"]
+ "'"
)
)
assert privacy == {"argon2id": True, "digest_bytes": 32}
receipt["credential_storage"] = {
**privacy,
"raw_password_and_token_absent_in_dump": True,
}
cache("FLUSHDB")
assert (
client.get("/api/v1/users/me", headers=headers).status_code == 200
)
info_before = cache("INFO", "stats")
for _ in range(20):
assert (
client.get("/api/v1/users/me", headers=headers).status_code
== 200
)
info_after = cache("INFO", "stats")
def hits(value):
return int(
next(
line.split(":")[1]
for line in value.splitlines()
if line.startswith("keyspace_hits:")
)
)
assert hits(info_after) - hits(info_before) >= 20
receipt["cache_hits"] = hits(info_after) - hits(info_before)
compose("stop", "cache")
try:
started = time.perf_counter()
assert (
client.get("/api/v1/users/me", headers=headers).status_code
== 200
)
receipt["cache_stopped"] = {
"status": 200,
"seconds": time.perf_counter() - started,
}
finally:
compose("up", "-d", "--no-recreate", "--wait", "cache")
cache("FLUSHDB")
assert (
client.get(path, headers=headers).json()["items"][0]["id"]
== message_id
)
receipt["cache_flush_preserves_truth"] = True
for mode in ["pause", "stop"]:
# Дожидаемся окончания circuit backoff и доказываем cache hit
# непосредственно перед отказом источника истины.
for _ in range(30):
assert (
client.get("/api/v1/users/me", headers=headers).status_code
== 200
)
if cache("EXISTS", "user:" + user["id"]) == "1":
break
time.sleep(0.25)
else:
raise AssertionError("User cache not primed before PG fault")
compose(mode, "postgres")
try:
started = time.perf_counter()
result = client.get("/api/v1/users/me", headers=headers)
elapsed = time.perf_counter() - started
assert result.status_code == 503 and elapsed < 6
assert result.json()["error"]["code"] == "unavailable"
assert (
result.headers["x-request-id"]
== result.json()["error"]["request_id"]
)
assert client.get("/health/live").status_code == 200
receipt["postgres_" + mode] = {
"cache_card_present_before_failure": True,
"status": result.status_code,
"seconds": elapsed,
"cached_identity_not_trusted": True,
}
finally:
if mode == "pause":
compose("unpause", "postgres")
else:
compose("up", "-d", "--no-recreate", "--wait", "postgres")
ready(timeout=45)
assert (
client.get(path, headers=headers).json()["items"][0]["id"]
== message_id
)
# Отказ посередине вставки участников откатывает весь чат.
outsider, _, _ = actor(client)
title = "rollback_" + uuid4().hex
function = "rollback_" + uuid4().hex[:8]
sql(
"CREATE FUNCTION "
+ function
+ "() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN "
+ "IF NEW.user_id='"
+ outsider["id"]
+ "'::uuid THEN RAISE EXCEPTION 'synthetic member failure'; "
+ "END IF; RETURN NEW; END $$; CREATE TRIGGER "
+ function
+ " BEFORE INSERT ON members FOR EACH ROW EXECUTE FUNCTION "
+ function
+ "()"
)
try:
result = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": title,
"member_ids": [user["id"], outsider["id"]],
},
)
assert result.status_code == 500
assert (
sql("SELECT count(*) FROM chats WHERE title='" + title + "'")
== "0"
)
receipt["chat_transaction_rollback"] = {
"injected_failure_status": 500,
"remaining_chat_rows": 0,
}
finally:
sql(
"DROP TRIGGER "
+ function
+ " ON members; DROP FUNCTION "
+ function
+ "()"
)
# Короткий TTL — на отдельном API; обычный smoke сохраняет TTL3600.
name, base = extra_api(ttl=2)
try:
with httpx.Client(
base_url=base, trust_env=False, timeout=15
) as short:
_, _, expiring = actor(short)
assert (
short.get(
"/api/v1/users/me", headers=auth(expiring)
).status_code
== 200
)
time.sleep(2.2)
assert (
short.get(
"/api/v1/users/me", headers=auth(expiring)
).status_code
== 401
)
receipt["expiry"] = {"ttl_seconds": 2, "status_after_expiry": 401}
finally:
command(["docker", "rm", "-f", name])
assert (
client.delete(
"/api/v1/auth/sessions/current", headers=headers
).status_code
== 204
)
assert (
client.get("/api/v1/users/me", headers=headers).status_code == 401
)
receipt["logout_after_cache_hit"] = "PASS"
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(receipt, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+116
View File
@@ -0,0 +1,116 @@
import argparse
import json
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import actor, auth, command, extra_api, sql
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", required=True)
args = parser.parse_args()
output = Path(args.output)
if output.exists():
raise SystemExit("Choose new output path")
name, base = extra_api(pool_size=1)
function = "poolwait_" + uuid4().hex[:8]
trigger_created = False
try:
with httpx.Client(
base_url=base, trust_env=False, timeout=15
) as client:
user, _, session = actor(client)
headers = auth(session)
chat = client.post(
"/api/v1/chats",
headers=headers,
json={"title": "Pool test", "member_ids": [user["id"]]},
).json()
path = "/api/v1/chats/" + chat["id"] + "/messages"
sql(
"CREATE FUNCTION "
+ function
+ "() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='"
+ chat["id"]
+ "'::uuid THEN PERFORM pg_sleep(4); END IF; RETURN NEW; END $$; CREATE TRIGGER "
+ function
+ " BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION "
+ function
+ "()"
)
trigger_created = True
with ThreadPoolExecutor(max_workers=1) as pool:
blocking = pool.submit(
client.post,
path,
headers=headers,
json={"text": "Synthetic delayed insert"},
)
for _ in range(30):
active = sql(
"SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND query LIKE 'INSERT INTO messages%'"
)
if int(active) > 0:
break
time.sleep(0.05)
else:
raise AssertionError("Blocked INSERT not observed")
started = time.perf_counter()
second = client.get("/api/v1/users/me", headers=headers)
waited = time.perf_counter() - started
assert second.status_code == 503 and 0.8 <= waited < 3
first = blocking.result(timeout=6)
assert first.status_code == 503
sql(
"DROP TRIGGER "
+ function
+ " ON messages; DROP FUNCTION "
+ function
+ "()"
)
trigger_created = False
assert client.get(path, headers=headers).json()["items"] == []
assert (
client.post(
path, headers=headers, json={"text": "After recovery"}
).status_code
== 201
)
result = {
"date": datetime.now(UTC).isoformat(),
"commit": command(["git", "rev-parse", "HEAD"]),
"command": "python -m scripts.verify_pool",
"pool_size": 1,
"blocked_insert_observed_in_pg_stat_activity": True,
"pool_wait_seconds": waited,
"pool_timeout_status": second.status_code,
"statement_timeout_status": first.status_code,
"failed_write_rolled_back": True,
"write_after_recovery": 201,
"result": "PASS",
}
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
)
print(json.dumps(result, ensure_ascii=False, indent=2))
finally:
if trigger_created:
sql(
"DROP TRIGGER IF EXISTS "
+ function
+ " ON messages; DROP FUNCTION IF EXISTS "
+ function
+ "()"
)
command(["docker", "rm", "-f", name])
if __name__ == "__main__":
main()
View File
+44
View File
@@ -0,0 +1,44 @@
import json
import logging
import os
import signal
import sys
def early_stop(signum, _frame):
print(
json.dumps({"event": "stopped_before_startup", "signal": signum}),
flush=True,
)
sys.exit(0)
signal.signal(signal.SIGTERM, early_stop)
signal.signal(signal.SIGINT, early_stop)
import uvicorn # noqa: E402
from src.logging_config import JsonFormatter # noqa: E402
def main() -> None:
host = os.environ.get("APP_HOST", "127.0.0.1")
port = int(os.environ.get("APP_PORT", "8080"))
if not 1 <= port <= 65535:
raise ValueError("APP_PORT must be between 1 and 65535")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
uvicorn.run(
"src.api:app",
host=host,
port=port,
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
log_config=None,
)
if __name__ == "__main__":
main()
+298
View File
@@ -0,0 +1,298 @@
import json
import logging
import time
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import datetime
from functools import partial
from uuid import UUID, uuid4
from anyio import to_thread
from fastapi import FastAPI, Request
from fastapi.encoders import jsonable_encoder
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from starlette.responses import JSONResponse, Response
from src.cache import UserCache
from src.cursor import parse_uuid
from src.database import Database
from src.domain import timestamp
from src.errors import DomainError, invalid
from src.password import Passwords
from src.repository import PostgresRepository
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
from src.service import MessengerService
from src.settings import Settings
logger = logging.getLogger("messenger")
def response(data: object, *, status: int = 200) -> JSONResponse:
return JSONResponse(
jsonable_encoder(
data, custom_encoder={datetime: timestamp, UUID: str}
),
status_code=status,
)
async def body(request: Request, model: type[Input]) -> Input:
try:
data = await request.body()
return model.model_validate(
json.loads(
data.decode("utf-8"),
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
)
)
except ValueError, ValidationError, UnicodeError, RecursionError:
raise invalid() from None
def pagination(request: Request) -> tuple[int, str | None]:
limits = request.query_params.getlist("limit")
cursors = request.query_params.getlist("cursor")
if len(limits) > 1 or len(cursors) > 1:
raise invalid()
raw = limits[0] if limits else "50"
if not raw.isascii() or not raw.isdecimal():
raise invalid()
raw = raw.lstrip("0") or "0"
if len(raw) > 3:
raise invalid()
limit = int(raw)
if not 1 <= limit <= 100:
raise invalid()
return limit, cursors[0] if cursors else None
def service(request: Request) -> MessengerService:
return request.app.state.service
async def run_io(function, **kwargs):
return await to_thread.run_sync(partial(function, **kwargs))
async def actor(request: Request):
return await run_io(
service(request).identify, value=request.headers.get("authorization")
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
settings = Settings.load()
database = Database(url=settings.database_url, size=settings.pool_size)
cache = UserCache(url=settings.cache_url)
try:
await run_io(database.open)
passwords = await run_io(Passwords)
app.state.database = database
app.state.service = MessengerService(
repository=PostgresRepository(database=database, cache=cache),
passwords=passwords,
session_ttl=settings.session_ttl,
)
app.state.ready = True
yield
finally:
app.state.ready = False
await run_io(cache.close)
await run_io(database.close)
app = FastAPI(
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
)
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid4())
request.state.request_id = request_id
started = time.perf_counter()
try:
result = await call_next(request)
except Exception:
logger.error(
json.dumps({"event": "internal_error", "request_id": request_id})
)
result = response(
{
"error": {
"code": "internal_error",
"message": "Internal error",
"request_id": request_id,
}
},
status=500,
)
result.headers["X-Request-Id"] = request_id
route = request.scope.get("route")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request_id,
"method": request.method,
"route": getattr(route, "path", "unmatched"),
"status": result.status_code,
"elapsed_ms": round((time.perf_counter() - started) * 1000, 3),
}
)
)
return result
@app.exception_handler(DomainError)
async def domain_error(request: Request, error: DomainError):
result = response(
{
"error": {
"code": error.code,
"message": error.message,
"request_id": request.state.request_id,
}
},
status=error.status,
)
result.headers.update(error.headers)
return result
@app.exception_handler(HTTPException)
async def http_error(request: Request, error: HTTPException):
code = "not_found" if error.status_code == 404 else "invalid_request"
return response(
{
"error": {
"code": code,
"message": "Resource not found"
if error.status_code == 404
else "Invalid request",
"request_id": request.state.request_id,
}
},
status=error.status_code,
)
@app.get("/health/live")
async def live():
return response({"status": "ok"})
@app.get("/health/ready")
async def ready():
if not getattr(app.state, "ready", False):
raise DomainError(503, "unavailable", "Not ready")
await run_io(app.state.database.ping)
return response({"status": "ok"})
@app.post("/api/v1/users")
async def register(request: Request):
data = await body(request, CreateUser)
return response(
await run_io(service(request).register, **data.model_dump()),
status=201,
)
@app.get("/api/v1/users/me")
async def current(request: Request):
return response(await actor(request))
@app.post("/api/v1/chats")
async def create_chat(request: Request):
user = await actor(request)
data = await body(request, CreateChat)
return response(
await run_io(
service(request).create_chat, actor_id=user.id, **data.model_dump()
),
status=201,
)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).chats,
actor_id=user.id,
limit=limit,
cursor=cursor,
)
)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(request: Request, chat_id: str):
user = await actor(request)
return response(
await run_io(
service(request).get_chat,
actor_id=user.id,
chat_id=parse_uuid(chat_id),
)
)
@app.post("/api/v1/chats/{chat_id}/messages")
async def send(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
data = await body(request, CreateMessage)
return response(
await run_io(
service(request).send,
actor_id=user.id,
chat_id=cid,
**data.model_dump(),
),
status=201,
)
@app.get("/api/v1/chats/{chat_id}/messages")
async def history(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).history,
actor_id=user.id,
chat_id=cid,
limit=limit,
cursor=cursor,
)
)
@app.post("/api/v1/auth/sessions")
async def login(request: Request):
return response(
await run_io(
service(request).login,
authorization=request.headers.get("authorization"),
),
status=201,
)
@app.delete("/api/v1/auth/sessions/current")
async def logout(request: Request):
await run_io(
service(request).logout,
authorization=request.headers.get("authorization"),
)
return Response(status_code=204)
+66
View File
@@ -0,0 +1,66 @@
import json
import threading
import time
from datetime import datetime
from uuid import UUID
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from src.domain import User, timestamp
class UserCache:
def __init__(self, *, url: str):
self.client = redis.Redis.from_url(
url,
socket_connect_timeout=0.15,
socket_timeout=0.15,
max_connections=16,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
self.disabled_until = 0.0
self.lock = threading.Lock()
def close(self):
self.client.close()
def _command(self, function, *args, **kwargs):
if time.monotonic() < self.disabled_until:
return None
try:
return function(*args, **kwargs)
except redis.RedisError, OSError:
with self.lock:
self.disabled_until = time.monotonic() + 5
return None
def get(self, *, user_id: UUID) -> User | None:
raw = self._command(self.client.get, "user:" + str(user_id))
if raw is None:
return None
try:
row = json.loads(raw)
if UUID(row["id"]) != user_id:
return None
return User(
user_id,
row["username"],
row["display_name"],
datetime.fromisoformat(row["created_at"]),
)
except ValueError, TypeError, KeyError:
return None
def put(self, *, user: User):
raw = json.dumps(
{
"id": str(user.id),
"username": user.username,
"display_name": user.display_name,
"created_at": timestamp(user.created_at),
}
)
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
+64
View File
@@ -0,0 +1,64 @@
import base64
import binascii
import json
import re
from datetime import datetime
from uuid import UUID
from src.domain import Key, timestamp
from src.errors import invalid
def parse_uuid(value: str) -> UUID:
if not isinstance(value, str) or not re.fullmatch(
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
):
raise invalid()
return UUID(value)
def encode_cursor(*, scope: str, key: Key) -> str:
payload = {
"v": 1,
"scope": scope,
"at": timestamp(key[0]),
"id": str(key[1]),
}
return (
base64.urlsafe_b64encode(
json.dumps(payload, separators=(",", ":")).encode()
)
.decode()
.rstrip("=")
)
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
if value is None:
return None
try:
if (
not value
or len(value) > 2048
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
):
raise ValueError
data = json.loads(
base64.b64decode(
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
)
)
if (
not isinstance(data, dict)
or type(data.get("v")) is not int
or data["v"] != 1
or data.get("scope") != scope
or not isinstance(data.get("at"), str)
):
raise ValueError
at = datetime.fromisoformat(data["at"])
if at.tzinfo is None or timestamp(at) != data["at"]:
raise ValueError
return at, parse_uuid(data["id"])
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
raise invalid() from None
+69
View File
@@ -0,0 +1,69 @@
import time
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from src.errors import DomainError
class DeadlineConnection(psycopg.Connection):
"""Клиентский срок действует и при замороженном PostgreSQL."""
def wait(self, gen, interval=0.1, timeout=None):
deadline = getattr(self, "request_deadline", None)
remaining = (
max(0.001, deadline - time.monotonic()) if deadline else 3.0
)
bound = min(remaining, timeout) if timeout is not None else remaining
try:
return super().wait(gen, interval=interval, timeout=bound)
except psycopg.OperationalError:
self.close()
raise
class Database:
def __init__(self, *, url: str, size: int):
self.pool = ConnectionPool(
url,
connection_class=DeadlineConnection,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": (
"-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=6000"
),
},
min_size=1,
max_size=size,
timeout=1,
open=False,
)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
self.pool.close(timeout=2)
@contextmanager
def connection(self):
try:
with self.pool.connection(timeout=1) as conn:
conn.request_deadline = time.monotonic() + 4
try:
yield conn
finally:
conn.request_deadline = None
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
raise DomainError(
503, "unavailable", "Database unavailable"
) from None
def ping(self):
with self.connection() as conn:
conn.execute("SELECT 1").fetchone()
+44
View File
@@ -0,0 +1,44 @@
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
def utcnow() -> datetime:
return datetime.now(UTC)
def timestamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
type Record = User | Chat | Message
type Key = tuple[datetime, UUID]
def order_key(record: Record) -> Key:
return record.created_at, record.id
+16
View File
@@ -0,0 +1,16 @@
class DomainError(Exception):
def __init__(
self, status: int, code: str, message: str, headers=None
) -> None:
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid() -> DomainError:
return DomainError(400, "invalid_request", "Invalid request")
def not_found() -> DomainError:
return DomainError(404, "not_found", "Resource not found")
+17
View File
@@ -0,0 +1,17 @@
import json
import logging
from datetime import UTC, datetime
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
message = record.getMessage()
try:
event = json.loads(message)
if not isinstance(event, dict):
event = {"event": message}
except ValueError:
event = {"event": message}
event["timestamp"] = datetime.now(UTC).isoformat()
event["level"] = record.levelname.lower()
return json.dumps(event, ensure_ascii=False)
+51
View File
@@ -0,0 +1,51 @@
import hashlib
import json
from pathlib import Path
import psycopg
from src.settings import secret
def main():
with psycopg.connect(
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
) as conn:
conn.execute("SELECT pg_advisory_lock(2301002)")
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
for path in sorted(Path("migrations").glob("*.sql")):
content = path.read_bytes()
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s",
(path.name,),
).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError(
"Applied migration checksum mismatch"
)
continue
with conn.transaction():
conn.execute(content.decode())
conn.execute(
"INSERT INTO schema_migrations(name,checksum) "
"VALUES(%s,%s)",
(path.name, checksum),
)
print(
json.dumps(
{"event": "migration_applied", "name": path.name}
)
)
finally:
conn.execute("SELECT pg_advisory_unlock(2301002)")
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
import threading
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.low_level import Type
from src.errors import DomainError
class Passwords:
def __init__(self):
self.hasher = PasswordHasher(
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
)
self.slots = threading.BoundedSemaphore(2)
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
def _run(self, function, *args):
if not self.slots.acquire(timeout=5):
raise DomainError(503, "unavailable", "Authentication busy")
try:
return function(*args)
finally:
self.slots.release()
def hash(self, *, password: str) -> str:
return self._run(self.hasher.hash, password)
def verify(self, *, password: str, encoded: str | None) -> bool:
try:
return (
self._run(self.hasher.verify, encoded or self.dummy, password)
and encoded is not None
)
except VerificationError, InvalidHashError:
return False
+203
View File
@@ -0,0 +1,203 @@
from uuid import UUID
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from src.cache import UserCache
from src.database import Database
from src.domain import Chat, Key, Message, User
from src.errors import DomainError, invalid, not_found
USER_COLUMNS = "id, username, display_name, created_at"
def user_dto(row) -> User:
return User(
row["id"], row["username"], row["display_name"], row["created_at"]
)
def chat_dto(row) -> Chat:
return Chat(
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
)
def message_dto(row) -> Message:
return Message(
row["id"],
row["chat_id"],
row["sender_id"],
row["text"],
row["created_at"],
)
class PostgresRepository:
def __init__(self, *, database: Database, cache: UserCache):
self.database = database
self.cache = cache
def add_user(self, *, user: User, password_hash: str) -> User:
try:
with self.database.connection() as conn:
conn.execute(
"INSERT INTO users(id,username,display_name,"
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
(
user.id,
user.username,
user.display_name,
password_hash,
user.created_at,
),
)
except UniqueViolation:
raise DomainError(
409, "conflict", "Username already exists"
) from None
return user
def get_user(self, *, user_id: UUID) -> User | None:
cached = self.cache.get(user_id=user_id)
if cached is not None:
return cached
with self.database.connection() as conn:
row = conn.execute(
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(user_id,),
).fetchone()
if row is None:
return None
user = user_dto(row)
self.cache.put(user=user)
return user
def credentials(self, *, username: str):
with self.database.connection() as conn:
row = conn.execute(
"SELECT "
+ USER_COLUMNS
+ ", password_hash FROM users WHERE username=%s",
(username,),
).fetchone()
if row is None:
return None, None
return user_dto(row), row["password_hash"]
def add_session(
self, *, digest: bytes, user_id: UUID, created_at, expires_at
):
with self.database.connection() as conn:
conn.execute(
"INSERT INTO sessions(token_digest,user_id,created_at,"
"expires_at) VALUES(%s,%s,%s,%s)",
(digest, user_id, created_at, expires_at),
)
def session_user(self, *, digest: bytes) -> UUID | None:
with self.database.connection() as conn:
row = conn.execute(
"SELECT user_id FROM sessions WHERE token_digest=%s "
"AND expires_at>clock_timestamp()",
(digest,),
).fetchone()
return row["user_id"] if row else None
def delete_session(self, *, digest: bytes):
with self.database.connection() as conn:
conn.execute(
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
)
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
members = chat.member_ids
if actor_id not in members or len(set(members)) != len(members):
raise invalid()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute(
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
(chat.id, chat.title, chat.created_at),
)
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO members(chat_id,user_id,position) "
"VALUES(%s,%s,%s)",
[
(chat.id, user_id, position)
for position, user_id in enumerate(members)
],
)
except ForeignKeyViolation:
raise invalid() from None
return chat
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
with self.database.connection() as conn:
row = conn.execute(
"SELECT c.id,c.title,c.created_at,"
"array_agg(m.user_id ORDER BY m.position) member_ids "
"FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
(chat_id, actor_id),
).fetchone()
if row is None:
raise not_found()
return chat_dto(row)
def add_message(self, *, message: Message) -> Message:
try:
with self.database.connection() as conn:
conn.execute(
"INSERT INTO messages(id,chat_id,sender_id,"
"text,created_at) "
"VALUES(%s,%s,%s,%s,%s)",
(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
),
)
except ForeignKeyViolation:
raise not_found() from None
return message
def list_chats(
self, *, actor_id: UUID, after: Key | None, limit: int
) -> list[Chat]:
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
parameters = (
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
"FROM members WHERE chat_id=c.id "
"ORDER BY position) member_ids "
"FROM chats c JOIN members own ON own.chat_id=c.id "
"WHERE own.user_id=%s "
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
parameters,
).fetchall()
return [chat_dto(row) for row in rows]
def list_messages(
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
) -> list[Message]:
condition = "AND (created_at,id)>(%s,%s)" if after else ""
parameters = (
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
"WHERE chat_id=%s "
+ condition
+ " ORDER BY created_at,id LIMIT %s",
parameters,
).fetchall()
return [message_dto(row) for row in rows]
+51
View File
@@ -0,0 +1,51 @@
import re
from typing import Annotated
from uuid import UUID
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
StrictStr,
field_validator,
)
from src.cursor import parse_uuid
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
class Input(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value: object) -> object:
if isinstance(value, str) and (
not value.strip() or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
):
raise ValueError("Invalid text")
return value
class CreateUser(Input):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
class CreateMessage(Input):
text: StrictStr = Field(min_length=1, max_length=4000)
+157
View File
@@ -0,0 +1,157 @@
import base64
import hashlib
import re
import secrets
from datetime import timedelta
from uuid import UUID, uuid4
from src.cursor import decode_cursor, encode_cursor
from src.domain import Chat, Message, Record, User, order_key, utcnow
from src.errors import DomainError
from src.password import Passwords
from src.repository import PostgresRepository
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
items = rows[:limit]
cursor = (
encode_cursor(scope=scope, key=order_key(items[-1]))
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": cursor}
def unauthorized(*, basic=False):
challenge = (
'Basic realm="messenger", charset="UTF-8"'
if basic
else 'Bearer realm="messenger"'
)
return DomainError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": challenge},
)
def bearer(value):
if not value:
raise unauthorized()
scheme, _, token = value.partition(" ")
if scheme.lower() != "bearer" or not re.fullmatch(
r"[A-Za-z0-9_-]{32,256}", token
):
raise unauthorized()
return token
def token_digest(value):
return hashlib.sha256(value.encode()).digest()
class MessengerService:
def __init__(
self,
*,
repository: PostgresRepository,
passwords: Passwords,
session_ttl: int,
) -> None:
self.repository = repository
self.passwords = passwords
self.session_ttl = session_ttl
def identify(self, *, value: str | None) -> User:
token = bearer(value)
user_id = self.repository.session_user(digest=token_digest(token))
user = self.repository.get_user(user_id=user_id) if user_id else None
if user is None:
raise unauthorized()
return user
def register(
self, *, username: str, display_name: str, password: str
) -> User:
encoded = self.passwords.hash(password=password)
return self.repository.add_user(
user=User(uuid4(), username, display_name, utcnow()),
password_hash=encoded,
)
def login(self, *, authorization: str | None):
try:
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic":
raise ValueError
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
username, separator, password = plain.partition(":")
if (
not separator
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
or not 12 <= len(password) <= 128
or not password.strip()
):
raise ValueError
except ValueError, UnicodeError:
raise unauthorized(basic=True) from None
user, password_hash = self.repository.credentials(username=username)
if not self.passwords.verify(password=password, encoded=password_hash):
raise unauthorized(basic=True)
now = utcnow()
expires = now + timedelta(seconds=self.session_ttl)
token = secrets.token_urlsafe(32)
self.repository.add_session(
digest=token_digest(token),
user_id=user.id,
created_at=now,
expires_at=expires,
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires,
"user": user,
}
def logout(self, *, authorization: str | None):
self.identify(value=authorization)
self.repository.delete_session(
digest=token_digest(bearer(authorization))
)
def create_chat(
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
) -> Chat:
return self.repository.add_chat(
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
actor_id=actor_id,
)
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
def send(self, *, actor_id: UUID, chat_id: UUID, text: str) -> Message:
return self.repository.add_message(
message=Message(uuid4(), chat_id, actor_id, text, utcnow())
)
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
scope = f"chats:{actor_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_chats(
actor_id=actor_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
def history(
self, *, actor_id: UUID, chat_id: UUID, limit: int, cursor: str | None
) -> dict:
self.get_chat(actor_id=actor_id, chat_id=chat_id)
scope = f"messages:{chat_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_messages(
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
+37
View File
@@ -0,0 +1,37 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str) -> str:
file = os.environ.get(name + "_FILE")
if file:
return Path(file).read_text().strip()
value = os.environ.get(name)
if not value:
raise ValueError("Missing required secret setting: " + name)
return value
def integer(name: str, default: int, low: int, high: int) -> int:
value = int(os.environ.get(name, str(default)))
if not low <= value <= high:
raise ValueError("Invalid setting: " + name)
return value
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 10
@classmethod
def load(cls):
return cls(
secret("DATABASE_URL"),
secret("CACHE_URL"),
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
integer("DB_POOL_SIZE", 10, 1, 32),
)
+111
View File
@@ -0,0 +1,111 @@
import json
import os
from pathlib import Path
from uuid import uuid4
import httpx
import pytest
from jsonschema import Draft202012Validator, FormatChecker
CONTRACT = json.loads(
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
)
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8082")
PASSWORD = "Synthetic-Password-🌍-123"
TOKENS = {}
def check(result: httpx.Response, status: int, schema: str) -> dict:
assert result.status_code == status, result.text
assert result.headers["x-request-id"]
assert result.headers["content-type"].startswith("application/json")
payload = result.json()
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
payload
)
if schema == "Error":
assert payload["error"]["request_id"] == result.headers["x-request-id"]
assert (
payload["error"]["code"]
== {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
500: "internal_error",
}[status]
)
return payload
def register(client: httpx.Client, **overrides) -> dict:
data = {
"username": "u_" + uuid4().hex[:24],
"display_name": "Алиса 🌍",
"password": PASSWORD,
}
data.update(overrides)
user = check(client.post("/api/v1/users", json=data), 201, "User")
session = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(data["username"], data["password"]),
),
201,
"Session",
)
TOKENS[user["id"]] = session["token"]
return user
@pytest.fixture
def client():
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as client:
yield client
@pytest.fixture
def actors(client):
return register(client), register(client), register(client)
@pytest.fixture
def chat(client, actors):
alice, bob, _ = actors
return check(
client.post(
"/api/v1/chats",
headers=headers(alice),
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
),
201,
"Chat",
)
def headers(actor: dict) -> dict:
return {"Authorization": "Bearer " + TOKENS[actor["id"]]}
def history(client, actor, chat_id, limit=7):
found, cursor = [], None
while True:
params = {"limit": limit}
if cursor:
params["cursor"] = cursor
page = check(
client.get(
f"/api/v1/chats/{chat_id}/messages",
headers=headers(actor),
params=params,
),
200,
"MessagePage",
)
found.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return found
+412
View File
@@ -0,0 +1,412 @@
import base64
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from uuid import UUID, uuid4
import httpx
import pytest
from conftest import BASE_URL, PASSWORD, check, headers, history, register
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
def test_health(client, path):
assert check(client.get(path), 200, "Health") == {"status": "ok"}
@pytest.mark.parametrize(
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
)
def test_identification(client, value):
check(
client.get(
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
),
401,
"Error",
)
def test_identity(client, actors):
assert (
check(
client.get("/api/v1/users/me", headers=headers(actors[0])),
200,
"User",
)
== actors[0]
)
@pytest.mark.parametrize(
"data",
[
{},
[],
None,
{"username": "ABC", "display_name": "A"},
{"username": "abc\n", "display_name": "A"},
{"username": 123, "display_name": "A"},
{"username": "abc", "display_name": " "},
{"username": "abc", "display_name": "x" * 101},
{"username": "abc", "display_name": "A", "extra": True},
{"username": "abc", "display_name": "\ud800"},
],
)
def test_invalid_user(client, data):
if isinstance(data, dict):
data = {"password": PASSWORD, **data}
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
@pytest.mark.parametrize(
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
)
def test_malformed_json(client, raw):
check(client.post("/api/v1/users", content=raw), 400, "Error")
def test_auth_before_json(client):
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
def test_duplicate_username(client, actors):
check(
client.post(
"/api/v1/users",
json={
"username": actors[0]["username"],
"display_name": "B",
"password": PASSWORD,
},
),
409,
"Error",
)
def test_unicode_codepoints(client, actors, chat):
user = register(client, display_name="🌍" * 100)
assert user["display_name"] == "🌍" * 100
text = "🌍" * 4000
result = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": text},
),
201,
"Message",
)
assert result["text"] == text
assert result["sender_id"] == actors[0]["id"]
@pytest.mark.parametrize(
"method,suffix,body",
[
("GET", "", None),
("GET", "/messages", None),
("POST", "/messages", {"text": "hello"}),
],
)
def test_acl(client, actors, chat, method, suffix, body):
outsider = headers(actors[2])
actual = check(
client.request(
method,
f"/api/v1/chats/{chat['id']}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
missing = check(
client.request(
method,
f"/api/v1/chats/{uuid4()}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
assert actual["error"]["message"] == missing["error"]["message"]
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
def test_bad_message(client, actors, chat, value):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": value},
),
400,
"Error",
)
def test_sender_not_client_controlled(client, actors, chat):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "Hi", "sender_id": actors[1]["id"]},
),
400,
"Error",
)
@pytest.mark.parametrize(
"case",
[
"duplicate",
"unknown",
"no_caller",
"empty",
"wrong_type",
"bad_uuid",
"too_many",
"blank_title",
],
)
def test_invalid_chat(client, actors, case):
a, b, _ = actors
members = {
"duplicate": [a["id"], a["id"]],
"unknown": [a["id"], str(uuid4())],
"no_caller": [b["id"]],
"empty": [],
"wrong_type": a["id"],
"bad_uuid": ["broken"],
"too_many": [str(uuid4()) for _ in range(101)],
"blank_title": [a["id"]],
}[case]
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={
"title": " " if case == "blank_title" else "Title",
"member_ids": members,
},
),
400,
"Error",
)
@pytest.mark.parametrize(
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
)
def test_bad_limit(client, actors, chat, limit):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"limit": limit},
),
400,
"Error",
)
@pytest.mark.parametrize(
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
)
def test_bad_cursor(client, actors, chat, cursor):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"cursor": cursor},
),
400,
"Error",
)
def test_pagination(client, actors, chat):
a, b, outsider = actors
sent = []
path = f"/api/v1/chats/{chat['id']}/messages"
for i in range(23):
sent.append(
check(
client.post(
path,
headers=headers(a if i % 2 else b),
json={"text": f"Msg {i}"},
),
201,
"Message",
)
)
got = history(client, a, chat["id"])
assert got == sorted(
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
)
first = check(
client.get(path, headers=headers(a), params={"limit": 1}),
200,
"MessagePage",
)
second = check(
client.get(
path,
headers=headers(b),
params={"limit": 22, "cursor": first["next_cursor"]},
),
200,
"MessagePage",
)
assert second["items"] == got[1:]
assert second["next_cursor"] is None
chats = check(
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
)
assert chats["items"] == []
other = check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": "Other", "member_ids": [a["id"]]},
),
201,
"Chat",
)
check(
client.get(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(a),
params={"cursor": first["next_cursor"]},
),
400,
"Error",
)
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
data = json.loads(
base64.urlsafe_b64decode(
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
)
)
data["at"] = 17
forged = (
base64.urlsafe_b64encode(json.dumps(data).encode())
.decode()
.rstrip("=")
)
check(
client.get(path, headers=headers(a), params={"cursor": forged}),
400,
"Error",
)
def test_chat_pagination_scope(client, actors):
a, b, _ = actors
created = []
for i in range(4):
created.append(
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
),
201,
"Chat",
)
)
page = check(
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
200,
"ChatPage",
)
assert page["items"] == created[:2]
last = check(
client.get(
"/api/v1/chats",
headers=headers(a),
params={"limit": 2, "cursor": page["next_cursor"]},
),
200,
"ChatPage",
)
assert last["items"] == created[2:]
assert last["next_cursor"] is None
check(
client.get(
"/api/v1/chats",
headers=headers(b),
params={"cursor": page["next_cursor"]},
),
400,
"Error",
)
def test_concurrent_messages(client, actors, chat):
barrier = threading.Barrier(10)
a = actors[0]
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(i):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
return [
check(
conn.post(
path, headers=headers(a), json={"text": f"{i}:{j}"}
),
201,
"Message",
)["id"]
for j in range(10)
]
with ThreadPoolExecutor(max_workers=10) as pool:
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
rows = history(client, a, chat["id"])
assert len(rows) == len(ids) == len(set(ids)) == 100
assert set(ids) == {row["id"] for row in rows}
assert len({row["text"] for row in rows}) == 100
def test_registration_race(client):
username = "race_" + uuid4().hex[:20]
barrier = threading.Barrier(10)
def worker(_):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
result = conn.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Test",
"password": PASSWORD,
},
)
check(
result,
result.status_code,
"User" if result.status_code == 201 else "Error",
)
return result.status_code
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(worker, range(10)))
assert statuses.count(201) == 1
assert statuses.count(409) == 9
def test_unknown_path(client):
check(client.get("/does-not-exist"), 404, "Error")
+26
View File
@@ -0,0 +1,26 @@
import json
from uuid import uuid4
import pytest
from conftest import check, headers
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
def test_only_utf8_json(client, encoding):
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
payload["password"] = "Synthetic-encoding-test-123"
result = client.post(
"/api/v1/users",
content=json.dumps(payload).encode(encoding),
headers={"Content-Type": "application/json"},
)
check(result, 400, "Error")
def test_leading_zero_limit(client, actors, chat):
result = client.get(
"/api/v1/chats/" + chat["id"] + "/messages",
headers=headers(actors[0]),
params={"limit": "0001"},
)
check(result, 200, "MessagePage")
+114
View File
@@ -0,0 +1,114 @@
from uuid import uuid4
import httpx
import pytest
from conftest import PASSWORD, TOKENS, check, headers, register
@pytest.mark.parametrize(
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
)
def test_password_validation(client, password):
check(
client.post(
"/api/v1/users",
json={
"username": "pwd_" + uuid4().hex[:20],
"display_name": "Test",
"password": password,
},
),
400,
"Error",
)
def test_full_utf8_password(client):
password = "🌍" * 127 + ":"
user = register(client, password=password)
assert (
check(
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
)
== user
)
@pytest.mark.parametrize(
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
)
def test_bad_basic(client, authorization):
result = client.post(
"/api/v1/auth/sessions",
headers={"Authorization": authorization} if authorization else {},
)
check(result, 401, "Error")
assert (
result.headers["www-authenticate"]
== 'Basic realm="messenger", charset="UTF-8"'
)
def test_wrong_password_and_unknown_user(client):
user = register(client)
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
result = client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
)
data = check(result, 401, "Error")
assert data["error"]["message"] == "Authentication required"
def test_logout_is_not_revived_by_cache(client):
user = register(client)
auth = headers(user)
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
assert result.status_code == 204 and result.content == b""
assert result.headers["x-request-id"]
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
check(
client.delete("/api/v1/auth/sessions/current", headers=auth),
401,
"Error",
)
def test_only_current_session_revoked(client):
user = register(client)
another = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(user["username"], PASSWORD),
),
201,
"Session",
)
assert another["token"] != TOKENS[user["id"]]
assert len(another["token"]) >= 32
assert (
client.delete(
"/api/v1/auth/sessions/current", headers=headers(user)
).status_code
== 204
)
result = client.get(
"/api/v1/users/me",
headers={"Authorization": "Bearer " + another["token"]},
)
assert check(result, 200, "User") == user
def test_bearer_source_and_ignored_user_header(client, actors):
alice, bob, _ = actors
auth = {**headers(alice), "X-User-Id": bob["id"]}
assert (
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
== alice["id"]
)
result = client.get(
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
)
check(result, 401, "Error")
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'
Generated
+1058
View File
File diff suppressed because it is too large Load Diff