forked from mtusi2026/mtusi2026lab2
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1a50a21226 | ||
|
|
e2532247cc | ||
|
|
5e7c93a036 | ||
|
|
a38c889df2 | ||
|
|
8e3e16a8a6 | ||
|
|
7466a09178 | ||
|
|
171b6de9c8 | ||
|
|
d978b8ff4d | ||
|
|
195d4abfce | ||
|
|
5fb1f19de1 | ||
|
|
2f0a8672be | ||
|
|
064421e6ee |
@@ -18,3 +18,16 @@ __pycache__
|
|||||||
node_modules
|
node_modules
|
||||||
evidence
|
evidence
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
**/__pycache__
|
||||||
|
**/*.py[cod]
|
||||||
|
.work
|
||||||
|
.pytest_cache
|
||||||
|
.ruff_cache
|
||||||
|
tests
|
||||||
|
student-tests
|
||||||
|
research
|
||||||
|
scripts
|
||||||
|
contracts
|
||||||
|
*.md
|
||||||
|
Makefile
|
||||||
|
compose.yaml
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
APP_HOST=0.0.0.0
|
APP_HOST=0.0.0.0
|
||||||
APP_PORT=8080
|
APP_PORT=8080
|
||||||
APP_ENV=lab
|
APP_ENV=lab
|
||||||
|
PUBLIC_PORT=8082
|
||||||
SESSION_TTL_SECONDS=3600
|
SESSION_TTL_SECONDS=3600
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
# Choose credentials locally. File paths contain no secret values.
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
DATABASE_URL_FILE=/run/secrets/database_url
|
||||||
|
|||||||
@@ -1,18 +1,22 @@
|
|||||||
## Лабораторная 2
|
## Лабораторная 2
|
||||||
|
|
||||||
Автор / группа: TODO
|
Автор / группа: Маслов / БВТ2301
|
||||||
Целевая оценка (3/4/5): TODO
|
Целевая оценка (3/4/5): 5
|
||||||
Трек 5, если заявлен: TODO
|
Трек 5: трек 2, планы запросов и восстановление
|
||||||
|
|
||||||
Что реализовано и что перенесено: TODO
|
Перенесён собственный HTTP API ЛР1; добавлены PostgreSQL/SQL-миграции,
|
||||||
Команды запуска из чистого clone: TODO
|
Argon2id, постоянные Bearer-сессии, Valkey cache, bounded pool/timeouts.
|
||||||
Результаты проверок и ссылка на REPORT.md: TODO
|
Запуск: uv sync --locked; make bootstrap; make up; make test test-student.
|
||||||
Ограничения / известные ошибки: TODO
|
Проверки: [REPORT.md](../REPORT.md), evidence/clean-clone.json;
|
||||||
Источники и использование ИИ: TODO
|
80 HTTP-тестов, реальные отказы/rollback, 101110 сообщений и backup/restore.
|
||||||
|
Ограничения: один host, без HA/TLS; snapshot не хранит поздние COMMIT.
|
||||||
|
Источники: контракт, PostgreSQL/Psycopg/Argon2/Valkey documentation.
|
||||||
|
Помощь ИИ использована при разборе требований, реализации и проверках;
|
||||||
|
результаты подтверждаются сохранёнными запусками.
|
||||||
|
|
||||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||||
- [ ] Контракт и публичные тесты не ослаблены.
|
- [x] Контракт и публичные тесты не ослаблены.
|
||||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: Application checks
|
||||||
|
on: [push, pull_request]
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
jobs:
|
||||||
|
application:
|
||||||
|
runs-on: ubuntu-24.04
|
||||||
|
env:
|
||||||
|
NO_PROXY: localhost,127.0.0.1
|
||||||
|
no_proxy: localhost,127.0.0.1
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.14.7'
|
||||||
|
- run: python -m pip install uv==0.13.7
|
||||||
|
- run: uv sync --locked
|
||||||
|
- run: make bootstrap lint check validate PYTHON=.venv/bin/python
|
||||||
|
- run: make up
|
||||||
|
- run: make test test-student
|
||||||
|
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
- run: make verify VERIFY_OUTPUT=.work/ci-lifecycle.json
|
||||||
|
- if: always()
|
||||||
|
run: docker compose down --volumes
|
||||||
@@ -28,5 +28,8 @@ coverage/
|
|||||||
.DS_Store
|
.DS_Store
|
||||||
.idea/
|
.idea/
|
||||||
.vscode/
|
.vscode/
|
||||||
|
.pytest_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.work/
|
||||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
3.14.7
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
|
||||||
|
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
|
||||||
|
COPY --from=uv /uv /usr/local/bin/uv
|
||||||
|
WORKDIR /app
|
||||||
|
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
|
||||||
|
COPY pyproject.toml uv.lock ./
|
||||||
|
RUN uv sync --locked --no-dev --no-install-project
|
||||||
|
|
||||||
|
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
|
||||||
|
ARG VCS_REF=unknown
|
||||||
|
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||||
|
WORKDIR /app
|
||||||
|
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
|
||||||
|
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||||
|
COPY --from=builder /app/.venv /app/.venv
|
||||||
|
COPY src /app/src
|
||||||
|
COPY migrations /app/migrations
|
||||||
|
USER 10001:10001
|
||||||
|
EXPOSE 8080
|
||||||
|
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
|
||||||
|
STOPSIGNAL SIGTERM
|
||||||
|
CMD ["python", "-m", "src"]
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
PYTHON ?= python3
|
PYTHON ?= python3
|
||||||
BASE_URL ?= http://localhost:8080
|
BASE_URL ?= http://127.0.0.1:8082
|
||||||
CA_FILE ?=
|
CA_FILE ?=
|
||||||
ACTION_SCRIPT ?=
|
ACTION_SCRIPT ?=
|
||||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||||
@@ -24,3 +24,27 @@ load:
|
|||||||
test-persistence:
|
test-persistence:
|
||||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
UV ?= uv
|
||||||
|
IMAGE ?= mtusi-messenger:lab2
|
||||||
|
VERIFY_OUTPUT ?= evidence/lifecycle-repeat.json
|
||||||
|
RESEARCH_OUTPUT ?= evidence/query-plan-repeat.json
|
||||||
|
|
||||||
|
.PHONY: bootstrap up down lint test-student image verify research
|
||||||
|
bootstrap:
|
||||||
|
$(PYTHON) scripts/bootstrap.py
|
||||||
|
up: bootstrap
|
||||||
|
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||||
|
down:
|
||||||
|
docker compose down
|
||||||
|
image:
|
||||||
|
docker build --build-arg VCS_REF=$$(git rev-parse HEAD) -t "$(IMAGE)" .
|
||||||
|
lint:
|
||||||
|
$(UV) run --locked ruff format src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py
|
||||||
|
$(UV) run --locked ruff check src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py
|
||||||
|
test-student:
|
||||||
|
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
|
||||||
|
verify:
|
||||||
|
$(UV) run --locked python -m scripts.verify_lifecycle --output "$(VERIFY_OUTPUT)"
|
||||||
|
research:
|
||||||
|
$(UV) run --locked python -m research.query_plan --output "$(RESEARCH_OUTPUT)"
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ЛР2 на 5 — план и гипотеза
|
||||||
|
|
||||||
|
Автор: Маслов, БВТ2301. Исходная ветка: master.
|
||||||
|
Перенесена только собственная реализация ЛР1, без её .git/contracts/tests.
|
||||||
|
|
||||||
|
## Архитектура до реализации запросов
|
||||||
|
|
||||||
|
API → сервисы → PostgreSQL repository, ограниченный sync pool.
|
||||||
|
Блокирующие сервисные операции выполняются в threadpool, не в event loop.
|
||||||
|
PostgreSQL — источник истины для users/chats/members/messages/sessions.
|
||||||
|
Valkey — disposable cache неизменяемого User, TTL30s; сессия/expiry/отзыв
|
||||||
|
всегда проверяются PostgreSQL. Отказ cache обходится; отказ PG даёт 503.
|
||||||
|
|
||||||
|
Таблицы: users(id PK, username UNIQUE, display_name, password_hash, time);
|
||||||
|
chats(id PK,title,time); members(chat,user PK/FK,position UNIQUE);
|
||||||
|
messages(id PK,chat,sender,text,time) с FK(chat,sender)→members;
|
||||||
|
sessions(token_digest PK,user FK,expires_at).
|
||||||
|
Индекс истории(chat_id,created_at,id), индекс members(user_id,chat_id).
|
||||||
|
UserDTO не содержит password_hash; raw session token никогда не хранится.
|
||||||
|
|
||||||
|
Транзакции: регистрация — одна INSERT с UNIQUE; чат+участники — одна;
|
||||||
|
сообщение — одна INSERT с FK; сессия и logout — отдельные durable записи.
|
||||||
|
Миграции версионируются SQL+checksum и блокируются advisory lock.
|
||||||
|
Не изменять применённые миграции. Compose запускает migrate перед API.
|
||||||
|
|
||||||
|
## Проверки и трек 2
|
||||||
|
|
||||||
|
Smoke, persistence с тем же Bearer, регистрационная гонка/rollback,
|
||||||
|
expiry2s на отдельном API, logout и неверный пароль, остановка cache/PG,
|
||||||
|
повторный migrate и bootstrap. Независимый clean clone и bundle.
|
||||||
|
|
||||||
|
Исследование на отдельной БД: >=100k сообщений в неравномерных чатах;
|
||||||
|
EXPLAIN ANALYZE BUFFERS после ANALYZE/прогрева, сравнение chat_id-only
|
||||||
|
и составного keyset-индекса, стоимость записи, WAL/размер индекса.
|
||||||
|
Backup pg_dump → новая БД pg_restore → count и выбранные полные строки.
|
||||||
|
|
||||||
|
Гипотеза до измерения: составной индекс уменьшает чтение середины
|
||||||
|
большого чата, но увеличивает размер/стоимость записи; величина неизвестна.
|
||||||
|
Дамп — snapshot, записи после snapshot без WAL архива могут потеряться.
|
||||||
|
Не выдавать SQL timings за HTTP latency и локальный backup за HA.
|
||||||
@@ -1,37 +1,270 @@
|
|||||||
# Отчёт — лабораторная 2
|
# Лабораторная 2 — PostgreSQL, сессии и Valkey
|
||||||
|
|
||||||
- Автор, группа: TODO
|
Автор: **Маслов, БВТ2301**. Цель: **5**, трек 2 «План запроса и восстановление».
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
Ветка `solution/bvt2301-maslov`; основа — skeleton ЛР2, ветка master.
|
||||||
- Commit/PR: TODO после публикации
|
Публикации и удалённого CI пока нет.
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Воспроизводимый запуск
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Нужны Docker 25+ с BuildKit/Compose v2+, make, Git и uv 0.13.7.
|
||||||
|
Python 3.14.7; FastAPI 0.143.3, Pydantic 2.14.1, Uvicorn 0.54.0,
|
||||||
|
psycopg 3.3.6, psycopg-pool 3.3.3, argon2-cffi 25.1.0, redis client 8.1.0.
|
||||||
|
Точные транзитивные версии — `uv.lock`. Базовые образы Python/uv,
|
||||||
|
PostgreSQL 18.3 и Valkey 8.1.3 закреплены тегами и digest.
|
||||||
|
|
||||||
## Архитектура и решения
|
```sh
|
||||||
|
uv sync --locked
|
||||||
|
make bootstrap
|
||||||
|
make check validate PYTHON=.venv/bin/python
|
||||||
|
make lint
|
||||||
|
make up
|
||||||
|
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 make test
|
||||||
|
make test-student
|
||||||
|
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \
|
||||||
|
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
make verify
|
||||||
|
uv run --locked python -m scripts.verify_pool --output evidence/pool-repeat.json
|
||||||
|
```
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
API: `http://127.0.0.1:8082`. PostgreSQL/Valkey не публикуют host-порты.
|
||||||
|
Для другого порта задать PUBLIC_PORT в окружении и передавать BASE_URL
|
||||||
|
проверкам. Все повторные evidence должны иметь новые имена файлов;
|
||||||
|
для make доступны VERIFY_OUTPUT и RESEARCH_OUTPUT.
|
||||||
|
|
||||||
## Доказательства критериев
|
`make down` останавливает стенд, сохраняя PostgreSQL volume. Его повторный
|
||||||
|
запуск использует прежние данные и секреты. Намеренное удаление **только
|
||||||
|
учебных данных данного проекта**: `docker compose down --volumes`.
|
||||||
|
Не применять к рабочим данным и не запускать глобальный Docker prune.
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
Bootstrap создаёт только отсутствующие `.env`/secret files. Каталог secrets
|
||||||
| --- | --- | --- | --- |
|
имеет 0700, файлы доступны non-root контейнеру через read-only file mounts.
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
Не пересоздавать пароль при сохранённом PG volume: initdb уже создал роль
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
со старым паролем. В Git/образ секреты и .env не попадают.
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
Конфигурация: APP_HOST/APP_PORT — bind внутри API; PUBLIC_PORT — Compose;
|
||||||
|
DATABASE_URL_FILE/CACHE_URL_FILE — пути к файловым секретам;
|
||||||
|
SESSION_TTL_SECONDS — абсолютный срок сессии (default 3600);
|
||||||
|
DB_POOL_SIZE — 1…32 (default 10). SESSION_STORE=postgres и APP_ENV в example
|
||||||
|
информационные: реализация не переключает источник истины этими строками.
|
||||||
|
Compose передаёт приложению только нужные ему настройки и секреты.
|
||||||
|
|
||||||
## Эксперименты и отказы
|
## Архитектура и перенос
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
Из своей ЛР1 перенесены HTTP-контракт, строгие схемы, ошибки/request ID,
|
||||||
|
ACL, UUID/UTC, cursor и собственные HTTP-тесты. Сохранены неизменёнными
|
||||||
|
README/COURSE/contracts/tests **текущего** skeleton. Git-история ЛР1
|
||||||
|
не переносилась; репозиторий ЛР2 независим.
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
```mermaid
|
||||||
| --- | --- | --- | --- |
|
flowchart LR
|
||||||
| TODO | TODO | TODO | TODO |
|
C[HTTP клиент] --> API[FastAPI: JSON, auth, request ID]
|
||||||
|
API --> T[Ограниченный threadpool: сервисные операции]
|
||||||
|
T --> PG[(PostgreSQL: данные и сессии)]
|
||||||
|
T --> V[(Valkey: карточка User, TTL 30 s)]
|
||||||
|
M[Одноразовый migrate] --> PG
|
||||||
|
```
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
SQL parameter binding используется для всех данных клиента. Динамические
|
||||||
|
идентификаторы в research — через psycopg.sql.Identifier и проверенный
|
||||||
|
UUID-префикс; в рабочем API пользователь не задаёт имена таблиц/SQL.
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
Модель и транзакции заданы до SQL в [PLAN.md](PLAN.md):
|
||||||
|
|
||||||
|
| Объект/операция | Ограничения и граница |
|
||||||
|
|---|---|
|
||||||
|
| users | UUID PK, UNIQUE username; KDF до получения соединения; INSERT атомарен |
|
||||||
|
| chats + members | FK, уникальный участник/позиция; чат и все участники в одной транзакции |
|
||||||
|
| messages | FK(chat_id,sender_id) на members; INSERT подтверждается после COMMIT |
|
||||||
|
| sessions | PK digest32bytes, FK user, created_at/expires_at; raw token не хранится |
|
||||||
|
| history | `(chat_id,created_at,id)`, tuple keyset + LIMIT limit+1 |
|
||||||
|
| migrations | SQL-файлы, checksum и advisory lock; отдельный сервис до API |
|
||||||
|
|
||||||
|
Миграции с нуля и повторное применение не создают пользовательские данные.
|
||||||
|
Изменение применённого файла отклоняется; добавляется новая миграция.
|
||||||
|
API сам схему не создаёт. Отказ migrate блокирует запуск API в Compose.
|
||||||
|
|
||||||
|
Пул на процесс: min 1 / max 10, ожидание 1 с; connect_timeout 2 с;
|
||||||
|
statement_timeout 2.5 с, lock_timeout 1 с, idle transaction timeout 6 с.
|
||||||
|
Общий клиентский срок операций на выданном соединении — 4 с, включая
|
||||||
|
замороженный сервер. При таймауте соединение закрывается и не возвращается
|
||||||
|
в пул как исправное. SQL, KDF и cache I/O не блокируют ASGI event loop.
|
||||||
|
Liveness не обращается к БД; readiness проверяет PostgreSQL, не Valkey.
|
||||||
|
|
||||||
|
## Пароли, сессии и кэш
|
||||||
|
|
||||||
|
- Argon2id: 64 MiB, t=3, p=4. Одновременно не больше двух KDF на процесс,
|
||||||
|
ожидание слота ограничено 5 с. UTF-8 пароль 12–128 code points обрабатывается
|
||||||
|
целиком, без усечения/нормализации. Проверено 127 emoji и двоеточие.
|
||||||
|
- Basic — только POST auth/sessions; собственный строгий base64/UTF-8 parser,
|
||||||
|
разделение по первому двоеточию. Неизвестный username проходит dummy KDF,
|
||||||
|
возвращается тот же 401. Хэш/пароль не входят в UserDTO и HTTP-ошибки.
|
||||||
|
- Токен — secrets.token_urlsafe(32), 256 бит энтропии; PostgreSQL хранит
|
||||||
|
только SHA-256 digest случайного токена. Это не password KDF и не JWT.
|
||||||
|
- Каждый запрос проверяет сессию и абсолютный expires_at **в PostgreSQL**.
|
||||||
|
Logout удаляет сессию; прежний cache hit не оживляет её. X-User-Id
|
||||||
|
игнорируется, Basic на бизнес-endpoint даёт 401. WWW-Authenticate
|
||||||
|
соответствует Basic/Bearer. Restart срок не продлевает.
|
||||||
|
- Valkey хранит только неизменяемую карточку User, TTL 30 с, maxmemory 64 MiB,
|
||||||
|
allkeys-lru; persistence выключена, собственного volume нет. Ошибка
|
||||||
|
cache — miss и чтение PG, backoff 5 с; socket/connect timeout 0.15 с,
|
||||||
|
скрытые retry отключены. При отказе PG cached User не авторизует запрос.
|
||||||
|
- API не изменяет карточки пользователей, поэтому инвалидации при записи
|
||||||
|
не требуется. При восстановлении старого snapshot вне API кэш надо
|
||||||
|
очистить. История вообще не кэшируется: read-after-write идёт из PG.
|
||||||
|
|
||||||
|
Образы API: non-root UID10001, exec Python PID1, healthcheck, read-only rootfs,
|
||||||
|
tmpfs и cap_drop=ALL. Лимиты API 1 CPU/512 MiB, PG 1 CPU/512 MiB, cache
|
||||||
|
0.25 CPU/128 MiB. Логи JSON: request ID/маршрут/код/время; без тел,
|
||||||
|
Authorization, паролей, токенов, их digest и параметров SQL.
|
||||||
|
|
||||||
|
## Таблица доказательств
|
||||||
|
|
||||||
|
| Критерий README | Команда | Факт / evidence |
|
||||||
|
|---|---|---|
|
||||||
|
| API ЛР1 + новый auth | `make test`, `make test-student` | Smoke PASS, 80 passed; [clean-clone-pre-pr.json](evidence/clean-clone-pre-pr.json) |
|
||||||
|
| Данные и прежний токен после restart всех stores | `make test-persistence`, `make verify` | [public-persistence.txt](evidence/public-persistence.txt), [lifecycle.json](evidence/lifecycle.json); CID/PID/StartedAt до/после |
|
||||||
|
| PK/FK/UNIQUE и атомарный чат | migrations, регистрационная гонка, injected rollback | [001_core.sql](migrations/001_core.sql), [lifecycle.json](evidence/lifecycle.json); после сбоя строка чата отсутствует |
|
||||||
|
| Версионируемые миграции и bootstrap | `make verify`, clean clone | Повтор без изменений, секреты сохранены, checksum mismatch отклонён; [clean-clone.json](evidence/clean-clone.json) |
|
||||||
|
| Реальная роль Valkey | `make verify` | 20 cache hits, stop/flush не теряют данные/сессию |
|
||||||
|
| Parameter binding, bounded pool и timeouts | `python -m scripts.verify_pool`, `make verify` | [pool.json](evidence/pool.json): pool1, ожидание 1.005 с →503; pause PG4.011 с →503 |
|
||||||
|
| Индекс, EXPLAIN ≥10k | `make research` | 90 планов для 5 размеров чата; [query-plan.json](evidence/query-plan.json) |
|
||||||
|
| Expiry, logout, неверный пароль, отказы | HTTP-тесты, lifecycle | TTL2s →401; stop cache →200; stop PG →503; logout →401 |
|
||||||
|
| Трек2: ≥100k, индексы/запись/backup | research | 101110 сообщений, 2 индекса, 3 повтора, восстановление и сверка |
|
||||||
|
| Независимый clean clone | `python scripts/verify_clean.py` | Полный PASS с новыми секретами/volumes, 80 passed; [clean-clone-pre-pr.json](evidence/clean-clone-pre-pr.json) |
|
||||||
|
|
||||||
|
Первая проверка не утверждает наличие карточки в cache непосредственно
|
||||||
|
перед PG fault без дополнительного EXISTS: [границы evidence](evidence/verification-notes.md).
|
||||||
|
Финальный вариант проверил это и неизменность expires_at.
|
||||||
|
|
||||||
|
## Финальный чистый clone
|
||||||
|
|
||||||
|
Последняя проверка перед PR: source commit
|
||||||
|
`5e7c93a0362e667551c8807a674fe04927b20e53`.
|
||||||
|
Новые `.env`, секреты и volumes; pinned image build; check/validate/lint;
|
||||||
|
публичный smoke; **80 passed за 93.23 с**; public persistence; lifecycle,
|
||||||
|
pool и checksum guard — PASS. Повторно выполнено исследование: 101110
|
||||||
|
сообщений, 90 EXPLAIN-планов, backup/restore snapshot совпал.
|
||||||
|
Хэши runtime-модулей совпали; image revision соответствует source commit.
|
||||||
|
Все временные containers/volumes этого clone удалены, рабочий volume сохранён.
|
||||||
|
|
||||||
|
Первая попытка clean-clone проверки остановилась на persistence: Docker
|
||||||
|
сменил автоматически назначенный host-порт при restart, а тест сохранил
|
||||||
|
прежний BASE_URL. Смена подтверждена на том же контейнере (65424 → 65441).
|
||||||
|
Проверка исправлена выбирать свободный **фиксированный** номер порта один
|
||||||
|
раз; код API/сессий не менялся. Первая попытка не выдана за успешную:
|
||||||
|
[описание](evidence/clean-attempt1.txt), [диагностика](evidence/port-restart.json).
|
||||||
|
Прежний полный повтор — [clean-clone.json](evidence/clean-clone.json).
|
||||||
|
Последний повтор перед PR —
|
||||||
|
[clean-clone-pre-pr.json](evidence/clean-clone-pre-pr.json).
|
||||||
|
|
||||||
|
После проверенного source commit менялись только отчёт, шаблон и evidence;
|
||||||
|
runtime src, зависимости, Dockerfile и Compose остались теми же.
|
||||||
|
Добавлен отдельный application workflow для CI; наличие файла не означает
|
||||||
|
его запуска на Gitea. Удалённый CI не заявляется.
|
||||||
|
Опубликован публичный fork `wh1tet3a/mtusi2026lab2`, ветка
|
||||||
|
`solution/bvt2301-maslov` → `mtusi2026/mtusi2026lab2:master`.
|
||||||
|
[PR №158: ЛР 2 — БВТ2301 — Маслов — на 5](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab2/pulls/158).
|
||||||
|
|
||||||
|
Повторная сверка выполнена по текущим README, COURSE, contracts/tests
|
||||||
|
и handoff от 30.10.2026. Все уровни 3/4 и выбранный трек 2 покрыты
|
||||||
|
таблицей доказательств. Исходные задание, контракт и публичные тесты
|
||||||
|
сохранены. Исправлены два пограничных дефекта: JSON принимается только
|
||||||
|
в UTF-8, десятичный `limit=0001` допустим. Три регрессионных теста сначала
|
||||||
|
упали на прежнем образе, затем прошли в новом чистом clone:
|
||||||
|
[исходные отказы](evidence/pre-pr-regressions.json).
|
||||||
|
При повторе также проверены неизменность абсолютного срока сессии,
|
||||||
|
наличие User в cache перед каждым отказом PG (503 за 4.044/0.011 с),
|
||||||
|
pool timeout 1.006 с, rollback, checksum миграции и backup/restore.
|
||||||
|
Исследовательские числа ниже относятся к исходному сохранённому
|
||||||
|
query-plan.json; новый clean clone независимо повторил методику и сверку,
|
||||||
|
но его timings не подставлялись в старые таблицы.
|
||||||
|
|
||||||
|
## Исследование: методика и результаты
|
||||||
|
|
||||||
|
Гипотеза в PLAN закоммичена до замера. Стенд: Apple M1, 8 CPU, 8 GiB RAM,
|
||||||
|
Docker Desktop 4.98.0/Engine29.9.2; PostgreSQL в контейнере 1 CPU/512 MiB.
|
||||||
|
Рабочие таблицы messenger не меняются: создаются research_UUID и restored_UUID
|
||||||
|
именно данного запуска, затем удаляются вместе с временным dump.
|
||||||
|
|
||||||
|
SQL seed: 5 чатов 90000/10000/1000/100/10, всего101110 сообщений;
|
||||||
|
уникальные тексты80ASCII. Seed обходит HTTP, это явно исследовательский
|
||||||
|
набор, а не нагрузка регистрации/Argon2. Схема и migration metadata
|
||||||
|
соответствуют приложению; исследовательский пользователь не используется
|
||||||
|
для входа. Данных других людей нет.
|
||||||
|
|
||||||
|
Сравниваются chat_id-only и `(chat_id,created_at,id)`. На каждом варианте
|
||||||
|
VACUUM ANALYZE и прогрев; head/middle/tail каждой истории, LIMIT 51,
|
||||||
|
EXPLAIN ANALYZE BUFFERS. Три повтора с порядком A/B, B/A, A/B; custom plans,
|
||||||
|
prepared statements в эксперименте отключены. Подготовительный OFFSET
|
||||||
|
применяется только для получения контрольной границы вне измеряемого
|
||||||
|
запроса; рабочая пагинация использует только keyset.
|
||||||
|
|
||||||
|
| Индекс | SQL middle90k, мс | Буферы | Размер, bytes | INSERT+COMMIT1000, мс | WAL, bytes |
|
||||||
|
|---|---:|---:|---:|---:|---:|
|
||||||
|
| chat_id | 7.237 | 2209 | 729088 | 37.53 | 347536 |
|
||||||
|
| keyset | 0.017 | 5 | 5980160 | 11.55 | 427672 |
|
||||||
|
|
||||||
|
Это медианы трёх повторов; min/max и остальные размеры/позиции —
|
||||||
|
[query-summary.csv](evidence/query-summary.csv), полные планы —
|
||||||
|
[query-plan.json](evidence/query-plan.json). График построен из raw и проверен:
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
Гипотеза чтения подтверждается. Цена — индекс примерно в 8.2 раза больше
|
||||||
|
и больший WAL на запись. Рост именно времени INSERT **не подтвердился**:
|
||||||
|
в этом опыте keyset быстрее и на записи. Причину этого различия отдельный
|
||||||
|
профиль не устанавливал. WAL — глобальный PG-счётчик; фон и cache state
|
||||||
|
могут влиять. EXPLAIN execution time не называется HTTP latency или CPU time.
|
||||||
|
|
||||||
|
Backup pg_dump -Fc: 3026259 bytes, 0.480 с; restore в новую БД: 0.664 с.
|
||||||
|
Сверены count всех 6 таблиц, SHA-256 всей истории и 15 полных сообщений.
|
||||||
|
После snapshot специально сделан ещё один SQL COMMIT: в источнике 101111,
|
||||||
|
в восстановленной копии 101110. Это демонстрация границы snapshot/RPO,
|
||||||
|
не измерение потери HTTP-подтверждений и не универсальный RTO сервиса.
|
||||||
|
Пик PG cgroup с момента старта контейнера 266039296 bytes (включает подготовку
|
||||||
|
и прочие проверки); это не RSS отдельного процесса и не пик только SELECT.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
uv run --locked python -m research.query_plan --output evidence/query-plan-repeat.json
|
||||||
|
uv run --locked python -m research.analyze evidence/query-plan-repeat.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Запускать последовательно с проверками отказов. Сырые результаты не
|
||||||
|
перезаписываются; dump не публикуется и не содержит рабочих данных.
|
||||||
|
|
||||||
|
## Вопросы защиты и ограничения
|
||||||
|
|
||||||
|
1. **Что подтверждает COMMIT?** Транзакция завершена и видна следующим
|
||||||
|
запросам согласно isolation. Проверены fsync=on, synchronous_commit=on,
|
||||||
|
full_page_writes=on. WAL flush защищает штатный durability сценарий,
|
||||||
|
но не потерю единственного диска/VM/host и не заменяет backup.
|
||||||
|
2. **Где сессии и что даст flush cache?** Сессии в PG, в cache только User.
|
||||||
|
Flush/restart cache даёт miss, но токен остаётся действующим; logout
|
||||||
|
действует через PG независимо от TTL карточки.
|
||||||
|
3. **Auth против ACL?** Сессия определяет пользователя. Отдельная проверка
|
||||||
|
membership определяет доступ к чату; посторонний получает такой же 404,
|
||||||
|
как отсутствующий ресурс. Успешный login сам по себе доступ не даёт.
|
||||||
|
4. **KDF и checksum?** Человеческий пароль имеет ограниченную энтропию,
|
||||||
|
поэтому нужен дорогой Argon2id с солью. Случайный 256-битный токен можно
|
||||||
|
индексировать быстрым SHA-256 digest; checksum backup решает проверку
|
||||||
|
целостности данных, а не защиту пароля.
|
||||||
|
|
||||||
|
Один PG/cache/API на host — не HA. Нет TLS на внешнем маршруте, стенд
|
||||||
|
только loopback и синтетические данные. PG init-role владеет учебной БД
|
||||||
|
и имеет широкие права; production разделение SQL-ролей не заявляется.
|
||||||
|
Backup snapshot не сохраняет последующие сообщения/отзывы сессий; без
|
||||||
|
WAL-архива и recovery policy это неизбежное ограничение восстановления.
|
||||||
|
POST сообщения ЛР2 не идемпотентен: после неизвестного исхода COMMIT
|
||||||
|
автоматический retry может создать дубль; это задача ЛР3.
|
||||||
|
|
||||||
|
## Источники и помощь
|
||||||
|
|
||||||
|
README/COURSE/contracts/tests преподавателя; справочные методические наборы;
|
||||||
|
[PostgreSQL](https://www.postgresql.org/docs/18/),
|
||||||
|
[Psycopg](https://www.psycopg.org/psycopg3/docs/),
|
||||||
|
[Argon2](https://argon2-cffi.readthedocs.io/),
|
||||||
|
[Valkey](https://valkey.io/topics/), FastAPI/Pydantic/Uvicorn.
|
||||||
|
Поведение client timeout дополнительно сверено с установленным кодом
|
||||||
|
psycopg3.3.6 и реальной паузой PostgreSQL.
|
||||||
|
Помощь ИИ использована при разборе требований, реализации, тестах и отчёте;
|
||||||
|
измерения подтверждены сохранёнными запусками.
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
name: mtusi-lab2-maslov
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: messenger
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||||
|
PGDATA: /var/lib/postgresql/data/pgdata
|
||||||
|
secrets: [postgres_password]
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 20
|
||||||
|
cpus: 1
|
||||||
|
mem_limit: 512m
|
||||||
|
memswap_limit: 512m
|
||||||
|
cache:
|
||||||
|
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||||
|
command: [valkey-server, /run/secrets/valkey.conf]
|
||||||
|
secrets:
|
||||||
|
- cache_password
|
||||||
|
- source: valkey_config
|
||||||
|
target: valkey.conf
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping']
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 20
|
||||||
|
cpus: 0.25
|
||||||
|
mem_limit: 128m
|
||||||
|
memswap_limit: 128m
|
||||||
|
migrate:
|
||||||
|
image: ${API_IMAGE:-mtusi-messenger:lab2}
|
||||||
|
pull_policy: never
|
||||||
|
command: [python, -m, src.migrate]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
secrets: [database_url]
|
||||||
|
depends_on:
|
||||||
|
postgres: {condition: service_healthy}
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 256m
|
||||||
|
read_only: true
|
||||||
|
api:
|
||||||
|
image: ${API_IMAGE:-mtusi-messenger:lab2}
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
args:
|
||||||
|
VCS_REF: ${VCS_REF:-unknown}
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${PUBLIC_PORT:-8082}:8080"
|
||||||
|
environment:
|
||||||
|
APP_HOST: 0.0.0.0
|
||||||
|
APP_PORT: 8080
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
CACHE_URL_FILE: /run/secrets/cache_url
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
|
||||||
|
secrets: [database_url, cache_url]
|
||||||
|
depends_on:
|
||||||
|
migrate: {condition: service_completed_successfully}
|
||||||
|
cache: {condition: service_healthy}
|
||||||
|
cpus: 1
|
||||||
|
mem_limit: 512m
|
||||||
|
memswap_limit: 512m
|
||||||
|
pids_limit: 128
|
||||||
|
read_only: true
|
||||||
|
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
|
||||||
|
cap_drop: [ALL]
|
||||||
|
security_opt: [no-new-privileges:true]
|
||||||
|
stop_grace_period: 10s
|
||||||
|
secrets:
|
||||||
|
postgres_password:
|
||||||
|
file: ./secrets/postgres_password
|
||||||
|
database_url:
|
||||||
|
file: ./secrets/database_url
|
||||||
|
cache_password:
|
||||||
|
file: ./secrets/cache_password
|
||||||
|
cache_url:
|
||||||
|
file: ./secrets/cache_url
|
||||||
|
valkey_config:
|
||||||
|
file: ./secrets/valkey.conf
|
||||||
|
name: valkey.conf
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
Source commit: 7466a09
|
||||||
|
Status: INCOMPLETE, not a successful final clean-clone verification.
|
||||||
|
|
||||||
|
The first isolated clone used PUBLIC_PORT=0 (automatic host port).
|
||||||
|
Bootstrap/build/static checks, smoke and HTTP tests completed before the
|
||||||
|
public persistence stage. After restart the public client kept its original
|
||||||
|
BASE_URL and could not recover within 90 seconds. Temporary project and
|
||||||
|
its volumes were removed by the cleanup handler.
|
||||||
|
|
||||||
|
Diagnostic reproduction checks whether Docker reassigns an automatic host
|
||||||
|
port on restart. Final verification selects an available numeric host port
|
||||||
|
once, so its BASE_URL remains valid through the restart hook.
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-07T05:47:11.012617+00:00",
|
||||||
|
"source_commit": "5e7c93a0362e667551c8807a674fe04927b20e53",
|
||||||
|
"fresh_secrets_environment_and_volumes": true,
|
||||||
|
"command": "python scripts/verify_clean.py",
|
||||||
|
"fixed_host_port_for_restart": 51761,
|
||||||
|
"static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\nuv run --locked ruff format src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\n27 files left unchanged\nuv run --locked ruff check src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\nAll checks passed!",
|
||||||
|
"internal_store_ports_not_published": true,
|
||||||
|
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:51761\" \nPASS: lab2 public API scenario; infrastructure and grading evidence remain separate.",
|
||||||
|
"http_tests": "........................................................................ [ 90%]\n........ [100%]\n80 passed in 93.23s (0:01:33)",
|
||||||
|
"public_persistence": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"http://127.0.0.1:51761\" \nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||||
|
"verify_lifecycle": {
|
||||||
|
"date": "2026-11-07T05:49:09.842951+00:00",
|
||||||
|
"commit": "5e7c93a0362e667551c8807a674fe04927b20e53",
|
||||||
|
"command": "python -m scripts.verify_lifecycle",
|
||||||
|
"runtime_source_hashes_match": true,
|
||||||
|
"image_revision": "5e7c93a0362e667551c8807a674fe04927b20e53",
|
||||||
|
"persistence": {
|
||||||
|
"absolute_expiry_unchanged": true,
|
||||||
|
"same_token": true,
|
||||||
|
"same_message": true,
|
||||||
|
"before": {
|
||||||
|
"api": {
|
||||||
|
"container": "52efca283e03",
|
||||||
|
"pid": 32752,
|
||||||
|
"started_at": "2026-11-07T05:49:05.222283053Z"
|
||||||
|
},
|
||||||
|
"postgres": {
|
||||||
|
"container": "4a8a9b88ac4b",
|
||||||
|
"pid": 32547,
|
||||||
|
"started_at": "2026-11-07T05:49:04.838047886Z"
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"container": "590272a808de",
|
||||||
|
"pid": 32569,
|
||||||
|
"started_at": "2026-11-07T05:49:04.863209469Z"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"after": {
|
||||||
|
"api": {
|
||||||
|
"container": "52efca283e03",
|
||||||
|
"pid": 33639,
|
||||||
|
"started_at": "2026-11-07T05:49:11.700302333Z"
|
||||||
|
},
|
||||||
|
"postgres": {
|
||||||
|
"container": "4a8a9b88ac4b",
|
||||||
|
"pid": 33460,
|
||||||
|
"started_at": "2026-11-07T05:49:11.381633375Z"
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"container": "590272a808de",
|
||||||
|
"pid": 33501,
|
||||||
|
"started_at": "2026-11-07T05:49:11.40642775Z"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"seconds": 5.98190225000144
|
||||||
|
},
|
||||||
|
"migration_repeat": "PASS",
|
||||||
|
"bootstrap_preserves_secrets": true,
|
||||||
|
"credential_storage": {
|
||||||
|
"argon2id": true,
|
||||||
|
"digest_bytes": 32,
|
||||||
|
"raw_password_and_token_absent_in_dump": true
|
||||||
|
},
|
||||||
|
"cache_hits": 20,
|
||||||
|
"cache_stopped": {
|
||||||
|
"status": 200,
|
||||||
|
"seconds": 0.07452341700263787
|
||||||
|
},
|
||||||
|
"cache_flush_preserves_truth": true,
|
||||||
|
"postgres_pause": {
|
||||||
|
"cache_card_present_before_failure": true,
|
||||||
|
"status": 503,
|
||||||
|
"seconds": 4.043605250000837,
|
||||||
|
"cached_identity_not_trusted": true
|
||||||
|
},
|
||||||
|
"postgres_stop": {
|
||||||
|
"cache_card_present_before_failure": true,
|
||||||
|
"status": 503,
|
||||||
|
"seconds": 0.01112437499978114,
|
||||||
|
"cached_identity_not_trusted": true
|
||||||
|
},
|
||||||
|
"chat_transaction_rollback": {
|
||||||
|
"injected_failure_status": 500,
|
||||||
|
"remaining_chat_rows": 0
|
||||||
|
},
|
||||||
|
"expiry": {
|
||||||
|
"ttl_seconds": 2,
|
||||||
|
"status_after_expiry": 401
|
||||||
|
},
|
||||||
|
"logout_after_cache_hit": "PASS",
|
||||||
|
"result": "PASS"
|
||||||
|
},
|
||||||
|
"verify_pool": {
|
||||||
|
"date": "2026-11-07T05:49:46.535430+00:00",
|
||||||
|
"commit": "5e7c93a0362e667551c8807a674fe04927b20e53",
|
||||||
|
"command": "python -m scripts.verify_pool",
|
||||||
|
"pool_size": 1,
|
||||||
|
"blocked_insert_observed_in_pg_stat_activity": true,
|
||||||
|
"pool_wait_seconds": 1.005813499999931,
|
||||||
|
"pool_timeout_status": 503,
|
||||||
|
"statement_timeout_status": 503,
|
||||||
|
"failed_write_rolled_back": true,
|
||||||
|
"write_after_recovery": 201,
|
||||||
|
"result": "PASS"
|
||||||
|
},
|
||||||
|
"applied_migration_checksum_guard": "PASS",
|
||||||
|
"research_repeated": {
|
||||||
|
"result": "PASS",
|
||||||
|
"messages": 101110,
|
||||||
|
"plans": 90,
|
||||||
|
"restored_snapshot_equal": true
|
||||||
|
},
|
||||||
|
"fsck": "",
|
||||||
|
"checks_completed": true,
|
||||||
|
"result": "PASS"
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-07T05:26:38.162144+00:00",
|
||||||
|
"source_commit": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
|
||||||
|
"fresh_secrets_environment_and_volumes": true,
|
||||||
|
"command": "python scripts/verify_clean.py",
|
||||||
|
"fixed_host_port_for_restart": 49287,
|
||||||
|
"static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n<home>/.local/bin/uv run --locked ruff format src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\n26 files left unchanged\n<home>/.local/bin/uv run --locked ruff check src student-tests research scripts/bootstrap.py scripts/stand.py scripts/verify_*.py\nAll checks passed!",
|
||||||
|
"internal_store_ports_not_published": true,
|
||||||
|
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:49287\" \nPASS: lab2 public API scenario; infrastructure and grading evidence remain separate.",
|
||||||
|
"http_tests": "........................................................................ [ 93%]\n..... [100%]\n77 passed in 86.00s (0:01:25)",
|
||||||
|
"public_persistence": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"http://127.0.0.1:49287\" \nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||||
|
"verify_lifecycle": {
|
||||||
|
"date": "2026-11-07T05:28:29.159541+00:00",
|
||||||
|
"commit": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
|
||||||
|
"command": "python -m scripts.verify_lifecycle",
|
||||||
|
"runtime_source_hashes_match": true,
|
||||||
|
"image_revision": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
|
||||||
|
"persistence": {
|
||||||
|
"absolute_expiry_unchanged": true,
|
||||||
|
"same_token": true,
|
||||||
|
"same_message": true,
|
||||||
|
"before": {
|
||||||
|
"api": {
|
||||||
|
"container": "4cc53c6190ac",
|
||||||
|
"pid": 89123,
|
||||||
|
"started_at": "2026-11-07T05:28:24.620912381Z"
|
||||||
|
},
|
||||||
|
"postgres": {
|
||||||
|
"container": "910b02e40597",
|
||||||
|
"pid": 88965,
|
||||||
|
"started_at": "2026-11-07T05:28:24.260037964Z"
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"container": "d47529714a4a",
|
||||||
|
"pid": 88983,
|
||||||
|
"started_at": "2026-11-07T05:28:24.279300006Z"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"after": {
|
||||||
|
"api": {
|
||||||
|
"container": "4cc53c6190ac",
|
||||||
|
"pid": 89990,
|
||||||
|
"started_at": "2026-11-07T05:28:30.971012968Z"
|
||||||
|
},
|
||||||
|
"postgres": {
|
||||||
|
"container": "910b02e40597",
|
||||||
|
"pid": 89812,
|
||||||
|
"started_at": "2026-11-07T05:28:30.640581301Z"
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"container": "d47529714a4a",
|
||||||
|
"pid": 89865,
|
||||||
|
"started_at": "2026-11-07T05:28:30.700865926Z"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"seconds": 5.304535000002943
|
||||||
|
},
|
||||||
|
"migration_repeat": "PASS",
|
||||||
|
"bootstrap_preserves_secrets": true,
|
||||||
|
"credential_storage": {
|
||||||
|
"argon2id": true,
|
||||||
|
"digest_bytes": 32,
|
||||||
|
"raw_password_and_token_absent_in_dump": true
|
||||||
|
},
|
||||||
|
"cache_hits": 20,
|
||||||
|
"cache_stopped": {
|
||||||
|
"status": 200,
|
||||||
|
"seconds": 0.013735125001403503
|
||||||
|
},
|
||||||
|
"cache_flush_preserves_truth": true,
|
||||||
|
"postgres_pause": {
|
||||||
|
"cache_card_present_before_failure": true,
|
||||||
|
"status": 503,
|
||||||
|
"seconds": 4.019069417001447,
|
||||||
|
"cached_identity_not_trusted": true
|
||||||
|
},
|
||||||
|
"postgres_stop": {
|
||||||
|
"cache_card_present_before_failure": true,
|
||||||
|
"status": 503,
|
||||||
|
"seconds": 0.005429041993920691,
|
||||||
|
"cached_identity_not_trusted": true
|
||||||
|
},
|
||||||
|
"chat_transaction_rollback": {
|
||||||
|
"injected_failure_status": 500,
|
||||||
|
"remaining_chat_rows": 0
|
||||||
|
},
|
||||||
|
"expiry": {
|
||||||
|
"ttl_seconds": 2,
|
||||||
|
"status_after_expiry": 401
|
||||||
|
},
|
||||||
|
"logout_after_cache_hit": "PASS",
|
||||||
|
"result": "PASS"
|
||||||
|
},
|
||||||
|
"verify_pool": {
|
||||||
|
"date": "2026-11-07T05:29:04.541663+00:00",
|
||||||
|
"commit": "8e3e16a8a6ea07f6a2b8615045735f812299e3d1",
|
||||||
|
"command": "python -m scripts.verify_pool",
|
||||||
|
"pool_size": 1,
|
||||||
|
"blocked_insert_observed_in_pg_stat_activity": true,
|
||||||
|
"pool_wait_seconds": 1.0053121249948163,
|
||||||
|
"pool_timeout_status": 503,
|
||||||
|
"statement_timeout_status": 503,
|
||||||
|
"failed_write_rolled_back": true,
|
||||||
|
"write_after_recovery": 201,
|
||||||
|
"result": "PASS"
|
||||||
|
},
|
||||||
|
"applied_migration_checksum_guard": "PASS",
|
||||||
|
"research_repeated": {
|
||||||
|
"result": "PASS",
|
||||||
|
"messages": 101110,
|
||||||
|
"plans": 90,
|
||||||
|
"restored_snapshot_equal": true
|
||||||
|
},
|
||||||
|
"fsck": "",
|
||||||
|
"checks_completed": true,
|
||||||
|
"result": "PASS"
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-07",
|
||||||
|
"source_commit": "2f0a867",
|
||||||
|
"public_smoke": "PASS: lab2 public API scenario",
|
||||||
|
"student_tests": {"passed": 77, "seconds": 91.82},
|
||||||
|
"command": "NO_PROXY=localhost,127.0.0.1 make test; pytest student-tests -q -x",
|
||||||
|
"note": "HTTP checks against real PostgreSQL/Valkey/API Compose stack; credentials remain in memory."
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-07T05:05:22.285580+00:00",
|
||||||
|
"commit": "5fb1f19de15552321d2c9f5449e088e6452a7189",
|
||||||
|
"command": "python -m scripts.verify_lifecycle",
|
||||||
|
"runtime_source_hashes_match": true,
|
||||||
|
"image_revision": "2f0a8672be67af9e62d2693c3df9bd683fcdae57",
|
||||||
|
"persistence": {
|
||||||
|
"same_token": true,
|
||||||
|
"same_message": true,
|
||||||
|
"before": {
|
||||||
|
"api": {
|
||||||
|
"container": "2b25734b0954",
|
||||||
|
"pid": 1409,
|
||||||
|
"started_at": "2026-11-07T05:00:09.733545264Z"
|
||||||
|
},
|
||||||
|
"postgres": {
|
||||||
|
"container": "29e277263007",
|
||||||
|
"pid": 1109,
|
||||||
|
"started_at": "2026-11-07T05:00:04.997609011Z"
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"container": "9f6a93f4f941",
|
||||||
|
"pid": 1041,
|
||||||
|
"started_at": "2026-11-07T05:00:04.881758678Z"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"after": {
|
||||||
|
"api": {
|
||||||
|
"container": "2b25734b0954",
|
||||||
|
"pid": 21469,
|
||||||
|
"started_at": "2026-11-07T05:05:24.139344714Z"
|
||||||
|
},
|
||||||
|
"postgres": {
|
||||||
|
"container": "29e277263007",
|
||||||
|
"pid": 21288,
|
||||||
|
"started_at": "2026-11-07T05:05:23.819680089Z"
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"container": "9f6a93f4f941",
|
||||||
|
"pid": 21329,
|
||||||
|
"started_at": "2026-11-07T05:05:23.858910756Z"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"seconds": 5.230442958010826
|
||||||
|
},
|
||||||
|
"migration_repeat": "PASS",
|
||||||
|
"bootstrap_preserves_secrets": true,
|
||||||
|
"credential_storage": {
|
||||||
|
"argon2id": true,
|
||||||
|
"digest_bytes": 32,
|
||||||
|
"raw_password_and_token_absent_in_dump": true
|
||||||
|
},
|
||||||
|
"cache_hits": 20,
|
||||||
|
"cache_stopped": {
|
||||||
|
"status": 200,
|
||||||
|
"seconds": 0.11006329099473078
|
||||||
|
},
|
||||||
|
"cache_flush_preserves_truth": true,
|
||||||
|
"postgres_pause": {
|
||||||
|
"status": 503,
|
||||||
|
"seconds": 4.011241999993217,
|
||||||
|
"cached_identity_not_trusted": true
|
||||||
|
},
|
||||||
|
"postgres_stop": {
|
||||||
|
"status": 503,
|
||||||
|
"seconds": 0.003582208009902388,
|
||||||
|
"cached_identity_not_trusted": true
|
||||||
|
},
|
||||||
|
"chat_transaction_rollback": {
|
||||||
|
"injected_failure_status": 500,
|
||||||
|
"remaining_chat_rows": 0
|
||||||
|
},
|
||||||
|
"expiry": {
|
||||||
|
"ttl_seconds": 2,
|
||||||
|
"status_after_expiry": 401
|
||||||
|
},
|
||||||
|
"logout_after_cache_hit": "PASS",
|
||||||
|
"result": "PASS"
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-07T05:15:33.380498+00:00",
|
||||||
|
"commit": "171b6de9c87c14cdfb1f9ec6a58aab7101c23d6a",
|
||||||
|
"command": "python -m scripts.verify_pool",
|
||||||
|
"pool_size": 1,
|
||||||
|
"blocked_insert_observed_in_pg_stat_activity": true,
|
||||||
|
"pool_wait_seconds": 1.0045806250127498,
|
||||||
|
"pool_timeout_status": 503,
|
||||||
|
"statement_timeout_status": 503,
|
||||||
|
"failed_write_rolled_back": true,
|
||||||
|
"write_after_recovery": 201,
|
||||||
|
"result": "PASS"
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-07",
|
||||||
|
"scope": "own temporary API, same container before and after docker restart",
|
||||||
|
"requested_host_port": "automatic",
|
||||||
|
"before": 65424,
|
||||||
|
"after": 65441,
|
||||||
|
"port_changed": true,
|
||||||
|
"result": "CONFIRMED",
|
||||||
|
"fix": "Choose a free numeric host port once for the clean-clone Compose project. Keep BASE_URL stable through the public persistence hook."
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"result": "EXPECTED_FAILURE_BEFORE_FIX",
|
||||||
|
"checks": {
|
||||||
|
"utf16_json": "201, expected 400",
|
||||||
|
"utf32_json": "201, expected 400",
|
||||||
|
"leading_zero_limit": "400, expected 200"
|
||||||
|
},
|
||||||
|
"command": "BASE_URL=<old-image-url> .venv/bin/python -m pytest student-tests/test_contract_edges.py -q",
|
||||||
|
"old_revision": "2f0a8672be67af9e62d2693c3df9bd683fcdae57"
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
Date: 2026-11-07
|
||||||
|
Runtime source: 2f0a8672be67af9e62d2693c3df9bd683fcdae57
|
||||||
|
Command: make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
|
||||||
|
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
|
||||||
|
Actual API/PostgreSQL/Valkey restart evidence: lifecycle.json
|
||||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 82 KiB |
@@ -0,0 +1,31 @@
|
|||||||
|
messages,position,variant,repeats,median_sql_ms,min_sql_ms,max_sql_ms,median_buffers
|
||||||
|
10,head,chat_only,3,0.011,0.011,0.015,3
|
||||||
|
10,head,keyset,3,0.006,0.006,0.008,4
|
||||||
|
10,middle,chat_only,3,0.01,0.008,0.013,3
|
||||||
|
10,middle,keyset,3,0.005,0.005,0.007,4
|
||||||
|
10,tail,chat_only,3,0.011,0.01,0.011,3
|
||||||
|
10,tail,keyset,3,0.006,0.006,0.007,4
|
||||||
|
100,head,chat_only,3,0.038,0.034,0.039,6
|
||||||
|
100,head,keyset,3,0.016,0.016,0.018,7
|
||||||
|
100,middle,chat_only,3,0.03,0.026,0.034,6
|
||||||
|
100,middle,keyset,3,0.015,0.014,0.015,5
|
||||||
|
100,tail,chat_only,3,0.028,0.027,0.031,6
|
||||||
|
100,tail,keyset,3,0.016,0.015,0.016,6
|
||||||
|
1000,head,chat_only,3,0.197,0.195,0.205,25
|
||||||
|
1000,head,keyset,3,0.018,0.016,0.02,5
|
||||||
|
1000,middle,chat_only,3,0.152,0.136,0.158,25
|
||||||
|
1000,middle,keyset,3,0.026,0.017,0.032,6
|
||||||
|
1000,tail,chat_only,3,0.092,0.08,0.106,25
|
||||||
|
1000,tail,keyset,3,0.016,0.015,0.019,6
|
||||||
|
10000,head,chat_only,3,1.801,1.755,1.845,224
|
||||||
|
10000,head,keyset,3,0.019,0.018,0.019,6
|
||||||
|
10000,middle,chat_only,3,1.187,1.17,1.198,224
|
||||||
|
10000,middle,keyset,3,0.016,0.016,0.018,5
|
||||||
|
10000,tail,chat_only,3,0.624,0.603,0.878,224
|
||||||
|
10000,tail,keyset,3,0.016,0.016,0.017,6
|
||||||
|
90000,head,chat_only,3,16.501,9.887,22.44,2209
|
||||||
|
90000,head,keyset,3,0.02,0.019,0.021,5
|
||||||
|
90000,middle,chat_only,3,7.237,7.082,8.505,2209
|
||||||
|
90000,middle,keyset,3,0.017,0.016,0.018,5
|
||||||
|
90000,tail,chat_only,3,4.287,4.282,4.674,2194
|
||||||
|
90000,tail,keyset,3,0.016,0.016,0.018,5
|
||||||
|
@@ -0,0 +1,20 @@
|
|||||||
|
# Планы запросов и восстановление
|
||||||
|
|
||||||
|
Три повтора, 101110 сообщений в 5 неравномерных чатах.
|
||||||
|
|
||||||
|
| Индекс | SQL середины чата90k, мс | Буферы | Размер, bytes | INSERT+COMMIT1000, мс | WAL, bytes |
|
||||||
|
|---|---:|---:|---:|---:|---:|
|
||||||
|
| chat_only | 7.237 | 2209 | 729088 | 37.53 | 347536 |
|
||||||
|
| keyset | 0.017 | 5 | 5980160 | 11.55 | 427672 |
|
||||||
|
|
||||||
|
Dump: 0.480 s, 3026259 bytes.
|
||||||
|
Restore в новую БД: 0.664 s.
|
||||||
|
Проверены count всех таблиц, SHA-256 всей истории и 15 полных сообщений.
|
||||||
|
Одна отдельная SQL-запись после snapshot не восстановлена — ожидаемая
|
||||||
|
граница snapshot backup. Это не HTTP RPO и не гарантия HA.
|
||||||
|
|
||||||
|
Время EXPLAIN — SQL execution time, не HTTP latency. Цена записи включает
|
||||||
|
COMMIT 1000 строк; WAL — глобальный счётчик PG и может включать фон.
|
||||||
|
Индекс keyset больше и даёт больше WAL, но в этом опыте INSERT быстрее:
|
||||||
|
гипотеза роста именно времени записи не подтвердилась. Причина механизма
|
||||||
|
такого различия отдельно не профилировалась.
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# Область первых проверок
|
||||||
|
|
||||||
|
В lifecycle.json подтверждены restart всех трёх процессов, тот же токен,
|
||||||
|
expiry/logout, cache hits и bounded 503 при остановке/паузе PostgreSQL.
|
||||||
|
Однако наличие карточки в кэше непосредственно перед отказом PG отдельно
|
||||||
|
не проверялось: после рестарта cache могло ещё действовать окно backoff.
|
||||||
|
Поэтому cached_identity_not_trusted в этой первой записи не является
|
||||||
|
самостоятельным доказательством сохранённого cache hit при отказе PG.
|
||||||
|
|
||||||
|
Финальный скрипт усилен: ждёт окончания backoff, проверяет EXISTS карточки
|
||||||
|
перед каждым воздействием на PG и сравнивает абсолютный expires_at до/после
|
||||||
|
restart. Этот вариант успешно повторён в финальном чистом clone; результат
|
||||||
|
в clean-clone.json. Runtime API при этом не менялся.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||||
|
password_hash text NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||||
|
created_at timestamptz NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE members (
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||||
|
PRIMARY KEY (chat_id, user_id),
|
||||||
|
UNIQUE (chat_id, position)
|
||||||
|
);
|
||||||
|
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||||
|
sender_id uuid NOT NULL,
|
||||||
|
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||||
|
created_at timestamptz NOT NULL,
|
||||||
|
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
|
||||||
|
CREATE TABLE sessions (
|
||||||
|
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
created_at timestamptz NOT NULL,
|
||||||
|
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
|
||||||
|
);
|
||||||
|
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
[project]
|
||||||
|
name = "mtusi-messenger-lab2"
|
||||||
|
version = "1.0.0"
|
||||||
|
requires-python = ">=3.14,<3.15"
|
||||||
|
dependencies = [
|
||||||
|
"fastapi==0.143.3", "pydantic==2.14.1", "uvicorn==0.54.0",
|
||||||
|
"psycopg[binary,pool]==3.3.6", "argon2-cffi>=25,<26", "redis==8.1.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[dependency-groups]
|
||||||
|
dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validator==0.7.2", "psutil>=7,<8", "matplotlib>=3.10,<4"]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 79
|
||||||
|
target-version = "py314"
|
||||||
|
include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py", "scripts/verify_*.py"]
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = ["E", "F", "I", "B", "UP"]
|
||||||
|
|
||||||
|
[tool.ruff.per-file-target-version]
|
||||||
|
"scripts/*.py" = "py39"
|
||||||
|
|
||||||
|
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
|
||||||
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
"research/*.py" = ["E501"]
|
||||||
|
"scripts/*.py" = ["E501"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["student-tests"]
|
||||||
|
addopts = "-ra"
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import statistics
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import matplotlib
|
||||||
|
|
||||||
|
matplotlib.use("Agg")
|
||||||
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("input", type=Path)
|
||||||
|
args = parser.parse_args()
|
||||||
|
result = json.loads(args.input.read_text())
|
||||||
|
groups = defaultdict(list)
|
||||||
|
for row in result["reads"]:
|
||||||
|
groups[(row["chat_messages"], row["position"], row["variant"])].append(
|
||||||
|
row
|
||||||
|
)
|
||||||
|
rows = []
|
||||||
|
for (size, position, variant), values in sorted(groups.items()):
|
||||||
|
ms = [v["execution_ms"] for v in values]
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"messages": size,
|
||||||
|
"position": position,
|
||||||
|
"variant": variant,
|
||||||
|
"repeats": len(values),
|
||||||
|
"median_sql_ms": statistics.median(ms),
|
||||||
|
"min_sql_ms": min(ms),
|
||||||
|
"max_sql_ms": max(ms),
|
||||||
|
"median_buffers": statistics.median(
|
||||||
|
v["shared_hits"] + v["shared_reads"] for v in values
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
directory = args.input.parent
|
||||||
|
with (directory / "query-summary.csv").open("w") as stream:
|
||||||
|
writer = csv.DictWriter(
|
||||||
|
stream, fieldnames=list(rows[0]), lineterminator="\n"
|
||||||
|
)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(rows)
|
||||||
|
lines = [
|
||||||
|
"# Планы запросов и восстановление",
|
||||||
|
"",
|
||||||
|
"Три повтора, 101110 сообщений в 5 неравномерных чатах.",
|
||||||
|
"",
|
||||||
|
"| Индекс | SQL середины чата90k, мс | Буферы | Размер, bytes | INSERT+COMMIT1000, мс | WAL, bytes |",
|
||||||
|
"|---|---:|---:|---:|---:|---:|",
|
||||||
|
]
|
||||||
|
for variant in ["chat_only", "keyset"]:
|
||||||
|
read = next(
|
||||||
|
r
|
||||||
|
for r in rows
|
||||||
|
if r["messages"] == 90000
|
||||||
|
and r["position"] == "middle"
|
||||||
|
and r["variant"] == variant
|
||||||
|
)
|
||||||
|
writes = [r for r in result["writes"] if r["variant"] == variant]
|
||||||
|
index_bytes = statistics.median(
|
||||||
|
r["bytes"] for r in result["indexes"] if r["variant"] == variant
|
||||||
|
)
|
||||||
|
latency = statistics.median(
|
||||||
|
r["insert_and_commit_seconds"] * 1000 for r in writes
|
||||||
|
)
|
||||||
|
wal = statistics.median(r["wal_bytes"] for r in writes)
|
||||||
|
lines.append(
|
||||||
|
f"| {variant} | {read['median_sql_ms']:.3f} | {read['median_buffers']:.0f} | {index_bytes:.0f} | {latency:.2f} | {wal:.0f} |"
|
||||||
|
)
|
||||||
|
lines.extend(
|
||||||
|
[
|
||||||
|
"",
|
||||||
|
f"Dump: {result['backup_seconds']:.3f} s, {result['backup_bytes']} bytes.",
|
||||||
|
f"Restore в новую БД: {result['restore_seconds']:.3f} s.",
|
||||||
|
"Проверены count всех таблиц, SHA-256 всей истории и 15 полных сообщений.",
|
||||||
|
"Одна отдельная SQL-запись после snapshot не восстановлена — ожидаемая",
|
||||||
|
"граница snapshot backup. Это не HTTP RPO и не гарантия HA.",
|
||||||
|
"",
|
||||||
|
"Время EXPLAIN — SQL execution time, не HTTP latency. Цена записи включает",
|
||||||
|
"COMMIT 1000 строк; WAL — глобальный счётчик PG и может включать фон.",
|
||||||
|
"Индекс keyset больше и даёт больше WAL, но в этом опыте INSERT быстрее:",
|
||||||
|
"гипотеза роста именно времени записи не подтвердилась. Причина механизма",
|
||||||
|
"такого различия отдельно не профилировалась.",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
(directory / "query-summary.md").write_text("\n".join(lines) + "\n")
|
||||||
|
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
|
||||||
|
for variant in ["chat_only", "keyset"]:
|
||||||
|
selected = sorted(
|
||||||
|
(
|
||||||
|
r
|
||||||
|
for r in rows
|
||||||
|
if r["variant"] == variant and r["position"] == "middle"
|
||||||
|
),
|
||||||
|
key=lambda r: r["messages"],
|
||||||
|
)
|
||||||
|
sizes = [r["messages"] for r in selected]
|
||||||
|
axes[0].plot(
|
||||||
|
sizes,
|
||||||
|
[r["median_sql_ms"] for r in selected],
|
||||||
|
marker="o",
|
||||||
|
label=variant,
|
||||||
|
)
|
||||||
|
axes[1].plot(
|
||||||
|
sizes,
|
||||||
|
[r["median_buffers"] for r in selected],
|
||||||
|
marker="o",
|
||||||
|
label=variant,
|
||||||
|
)
|
||||||
|
for ax, ylabel in zip(
|
||||||
|
axes, ["SQL execution time, ms", "Shared buffers touched"], strict=True
|
||||||
|
):
|
||||||
|
ax.set_xscale("log")
|
||||||
|
ax.set_yscale("log")
|
||||||
|
ax.set_xlabel("Messages in chat")
|
||||||
|
ax.set_ylabel(ylabel)
|
||||||
|
ax.grid(alpha=0.3)
|
||||||
|
ax.legend()
|
||||||
|
fig.suptitle(
|
||||||
|
"PostgreSQL 1 CPU / 512 MiB; midpoint keyset page; median of 3"
|
||||||
|
)
|
||||||
|
fig.tight_layout()
|
||||||
|
fig.savefig(directory / "query-plans.png", dpi=150)
|
||||||
|
plt.close(fig)
|
||||||
|
print("\n".join(lines))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,442 @@
|
|||||||
|
"""Изолированная БД: планы, цена записи, backup/restore без рабочих данных."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import resource
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID, uuid4, uuid5
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg import sql
|
||||||
|
from psycopg.conninfo import make_conninfo
|
||||||
|
|
||||||
|
from src.settings import secret
|
||||||
|
|
||||||
|
NAMESPACE = UUID("2e52c91b-b28f-4990-988c-7e8e15c9f131")
|
||||||
|
USER = uuid5(NAMESPACE, "research-user")
|
||||||
|
ORIGIN = datetime(2026, 1, 1, tzinfo=UTC)
|
||||||
|
SIZES = [90_000, 10_000, 1000, 100, 10]
|
||||||
|
|
||||||
|
|
||||||
|
def connect(database):
|
||||||
|
return psycopg.connect(
|
||||||
|
make_conninfo(secret("DATABASE_URL"), dbname=database),
|
||||||
|
autocommit=True,
|
||||||
|
connect_timeout=3,
|
||||||
|
prepare_threshold=None,
|
||||||
|
options="-c statement_timeout=60000 -c timezone=UTC",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def safe_database(name):
|
||||||
|
import re
|
||||||
|
|
||||||
|
if not re.fullmatch(r"(?:research|restored)_[a-f0-9]{12}", name):
|
||||||
|
raise ValueError("Refusing database outside research scope")
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot(conn):
|
||||||
|
counts = {}
|
||||||
|
for table in [
|
||||||
|
"users",
|
||||||
|
"chats",
|
||||||
|
"members",
|
||||||
|
"messages",
|
||||||
|
"sessions",
|
||||||
|
"schema_migrations",
|
||||||
|
]:
|
||||||
|
counts[table] = conn.execute(
|
||||||
|
sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table))
|
||||||
|
).fetchone()[0]
|
||||||
|
checksum = hashlib.sha256()
|
||||||
|
with conn.transaction(), conn.cursor(name="snapshot_messages") as cursor:
|
||||||
|
cursor.execute(
|
||||||
|
"SELECT id,chat_id,sender_id,text,created_at "
|
||||||
|
"FROM messages ORDER BY id"
|
||||||
|
)
|
||||||
|
for row in cursor:
|
||||||
|
checksum.update(
|
||||||
|
json.dumps(
|
||||||
|
[str(value) for value in row], separators=(",", ":")
|
||||||
|
).encode()
|
||||||
|
)
|
||||||
|
checksum.update(b"\n")
|
||||||
|
selected = []
|
||||||
|
for index, size in enumerate(SIZES):
|
||||||
|
chat = uuid5(NAMESPACE, "chat:" + str(index))
|
||||||
|
for offset in [0, size // 2, size - 1]:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT id,chat_id,sender_id,text,created_at "
|
||||||
|
"FROM messages WHERE chat_id=%s "
|
||||||
|
"ORDER BY created_at,id OFFSET %s LIMIT 1",
|
||||||
|
(chat, offset),
|
||||||
|
).fetchone()
|
||||||
|
selected.append([str(value) for value in row])
|
||||||
|
return {
|
||||||
|
"counts": counts,
|
||||||
|
"messages_sha256": checksum.hexdigest(),
|
||||||
|
"selected_messages": selected,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def inside_plan(database):
|
||||||
|
safe_database(database)
|
||||||
|
with connect("messenger") as admin:
|
||||||
|
admin.execute(
|
||||||
|
sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database))
|
||||||
|
)
|
||||||
|
with connect(database) as conn:
|
||||||
|
migration = Path("migrations/001_core.sql").read_bytes()
|
||||||
|
conn.execute(migration.decode())
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE schema_migrations (name text PRIMARY KEY, "
|
||||||
|
"checksum text NOT NULL, "
|
||||||
|
"applied_at timestamptz NOT NULL DEFAULT now())"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO schema_migrations(name,checksum) VALUES(%s,%s)",
|
||||||
|
("001_core.sql", hashlib.sha256(migration).hexdigest()),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO users VALUES(%s,%s,%s,%s,%s)",
|
||||||
|
(USER, "research_user", "Synthetic", "not-a-login", ORIGIN),
|
||||||
|
)
|
||||||
|
cursors = []
|
||||||
|
for index, size in enumerate(SIZES):
|
||||||
|
chat = uuid5(NAMESPACE, "chat:" + str(index))
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO chats VALUES(%s,%s,%s)",
|
||||||
|
(chat, "Research " + str(index), ORIGIN),
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO members VALUES(%s,%s,0)", (chat, USER))
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO messages SELECT "
|
||||||
|
"md5(%s || ':' || n::text)::uuid,%s,%s,"
|
||||||
|
"rpad('message ' || n,80,'x'),%s + n*interval '1 microsecond' "
|
||||||
|
"FROM generate_series(1,%s) n",
|
||||||
|
(str(chat), chat, USER, ORIGIN, size),
|
||||||
|
)
|
||||||
|
for position, offset in [
|
||||||
|
("head", 0),
|
||||||
|
("middle", size // 2),
|
||||||
|
("tail", max(0, size - 52)),
|
||||||
|
]:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT created_at,id FROM messages WHERE chat_id=%s "
|
||||||
|
"ORDER BY created_at,id OFFSET %s LIMIT 1",
|
||||||
|
(chat, offset),
|
||||||
|
).fetchone()
|
||||||
|
cursors.append((size, position, chat, row[0], row[1]))
|
||||||
|
assert (
|
||||||
|
conn.execute("SELECT count(*) FROM messages").fetchone()[0]
|
||||||
|
>= 100_000
|
||||||
|
)
|
||||||
|
query = (
|
||||||
|
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
|
||||||
|
"WHERE chat_id=%s AND (created_at,id)>(%s,%s) "
|
||||||
|
"ORDER BY created_at,id LIMIT 51"
|
||||||
|
)
|
||||||
|
reads, writes, indexes = [], [], []
|
||||||
|
for repeat, order in enumerate(
|
||||||
|
[
|
||||||
|
["chat_only", "keyset"],
|
||||||
|
["keyset", "chat_only"],
|
||||||
|
["chat_only", "keyset"],
|
||||||
|
],
|
||||||
|
1,
|
||||||
|
):
|
||||||
|
for variant in order:
|
||||||
|
conn.execute("DROP INDEX IF EXISTS messages_history_idx")
|
||||||
|
conn.execute("DROP INDEX IF EXISTS messages_chat_only_idx")
|
||||||
|
if variant == "chat_only":
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX messages_chat_only_idx "
|
||||||
|
"ON messages(chat_id)"
|
||||||
|
)
|
||||||
|
index_name = "messages_chat_only_idx"
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX messages_history_idx "
|
||||||
|
"ON messages(chat_id,created_at,id)"
|
||||||
|
)
|
||||||
|
index_name = "messages_history_idx"
|
||||||
|
conn.execute("VACUUM (ANALYZE) messages")
|
||||||
|
index_bytes = conn.execute(
|
||||||
|
"SELECT pg_relation_size(%s)", (index_name,)
|
||||||
|
).fetchone()[0]
|
||||||
|
indexes.append(
|
||||||
|
{
|
||||||
|
"repeat": repeat,
|
||||||
|
"variant": variant,
|
||||||
|
"bytes": index_bytes,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
for size, position, chat, at, mid in cursors:
|
||||||
|
parameters = (chat, at, mid)
|
||||||
|
conn.execute(query, parameters).fetchall()
|
||||||
|
plan = conn.execute(
|
||||||
|
"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query,
|
||||||
|
parameters,
|
||||||
|
).fetchone()[0][0]
|
||||||
|
reads.append(
|
||||||
|
{
|
||||||
|
"repeat": repeat,
|
||||||
|
"variant": variant,
|
||||||
|
"chat_messages": size,
|
||||||
|
"position": position,
|
||||||
|
"execution_ms": plan["Execution Time"],
|
||||||
|
"planning_ms": plan["Planning Time"],
|
||||||
|
"shared_hits": plan["Plan"].get(
|
||||||
|
"Shared Hit Blocks", 0
|
||||||
|
),
|
||||||
|
"shared_reads": plan["Plan"].get(
|
||||||
|
"Shared Read Blocks", 0
|
||||||
|
),
|
||||||
|
"plan": plan,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
# Окно записи включает COMMIT; cleanup за границей окна.
|
||||||
|
batch = "write:" + uuid4().hex
|
||||||
|
chat = uuid5(NAMESPACE, "chat:0")
|
||||||
|
lsn_before = conn.execute(
|
||||||
|
"SELECT pg_current_wal_insert_lsn()"
|
||||||
|
).fetchone()[0]
|
||||||
|
started = time.perf_counter()
|
||||||
|
with conn.transaction():
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO messages SELECT "
|
||||||
|
"md5(%s || ':' || n::text)::uuid,%s,%s,"
|
||||||
|
"rpad('write ' || n,80,'x'),%s + "
|
||||||
|
"n*interval '1 microsecond' "
|
||||||
|
"FROM generate_series(1,1000) n",
|
||||||
|
(batch, chat, USER, ORIGIN + timedelta(days=1)),
|
||||||
|
)
|
||||||
|
elapsed = time.perf_counter() - started
|
||||||
|
wal = conn.execute(
|
||||||
|
"SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),%s)",
|
||||||
|
(lsn_before,),
|
||||||
|
).fetchone()[0]
|
||||||
|
writes.append(
|
||||||
|
{
|
||||||
|
"repeat": repeat,
|
||||||
|
"variant": variant,
|
||||||
|
"inserted": 1000,
|
||||||
|
"insert_and_commit_seconds": elapsed,
|
||||||
|
"wal_bytes": int(wal),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM messages WHERE id IN "
|
||||||
|
"(SELECT md5(%s || ':' || n::text)::uuid "
|
||||||
|
"FROM generate_series(1,1000) n)",
|
||||||
|
(batch,),
|
||||||
|
)
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT count(*) FROM messages"
|
||||||
|
).fetchone()[0] == sum(SIZES)
|
||||||
|
result = {
|
||||||
|
"dataset": {
|
||||||
|
"chat_sizes": SIZES,
|
||||||
|
"messages": sum(SIZES),
|
||||||
|
"text_characters": 80,
|
||||||
|
"source": (
|
||||||
|
"synthetic SQL seed; no production API "
|
||||||
|
"or working tables modified"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"reads": reads,
|
||||||
|
"writes": writes,
|
||||||
|
"indexes": indexes,
|
||||||
|
"snapshot_before_backup": snapshot(conn),
|
||||||
|
"research_worker_maxrss_kib": resource.getrusage(
|
||||||
|
resource.RUSAGE_SELF
|
||||||
|
).ru_maxrss,
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def inside_after(database):
|
||||||
|
safe_database(database)
|
||||||
|
with connect(database) as conn:
|
||||||
|
value = uuid4()
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO messages VALUES(%s,%s,%s,%s,%s)",
|
||||||
|
(
|
||||||
|
value,
|
||||||
|
uuid5(NAMESPACE, "chat:0"),
|
||||||
|
USER,
|
||||||
|
"Committed after backup snapshot",
|
||||||
|
ORIGIN + timedelta(days=2),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"post_backup_message": str(value),
|
||||||
|
"count": conn.execute("SELECT count(*) FROM messages").fetchone()[
|
||||||
|
0
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def inside_snapshot(database):
|
||||||
|
safe_database(database)
|
||||||
|
with connect(database) as conn:
|
||||||
|
return snapshot(conn)
|
||||||
|
|
||||||
|
|
||||||
|
def inside_cleanup(databases):
|
||||||
|
with connect("messenger") as admin:
|
||||||
|
for database in databases:
|
||||||
|
safe_database(database)
|
||||||
|
admin.execute(
|
||||||
|
sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
|
||||||
|
sql.Identifier(database)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {"scoped_databases_removed": databases}
|
||||||
|
|
||||||
|
|
||||||
|
def outside(output):
|
||||||
|
from scripts.stand import ROOT, command, compose
|
||||||
|
|
||||||
|
if output.exists():
|
||||||
|
raise SystemExit("Choose a new output file")
|
||||||
|
identifier = uuid4().hex[:12]
|
||||||
|
database, restored = "research_" + identifier, "restored_" + identifier
|
||||||
|
dump_path = "/tmp/" + database + ".dump"
|
||||||
|
mount = str(ROOT / "research") + ":/app/research:ro"
|
||||||
|
|
||||||
|
def inside(phase, selected):
|
||||||
|
return json.loads(
|
||||||
|
compose(
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--no-deps",
|
||||||
|
"-v",
|
||||||
|
mount,
|
||||||
|
"api",
|
||||||
|
"python",
|
||||||
|
"-m",
|
||||||
|
"research.query_plan",
|
||||||
|
"--phase",
|
||||||
|
phase,
|
||||||
|
"--database",
|
||||||
|
selected,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
result = {
|
||||||
|
"date": datetime.now(UTC).isoformat(),
|
||||||
|
"commit": command(["git", "rev-parse", "HEAD"]),
|
||||||
|
"command": "python -m research.query_plan",
|
||||||
|
"db_cpu_limit": 1,
|
||||||
|
"db_memory_limit_bytes": 536870912,
|
||||||
|
"prepared_statements": "disabled for comparable custom EXPLAIN plans",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
result.update(inside("plan", database))
|
||||||
|
started = time.perf_counter()
|
||||||
|
compose(
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"postgres",
|
||||||
|
"pg_dump",
|
||||||
|
"-U",
|
||||||
|
"messenger",
|
||||||
|
"-Fc",
|
||||||
|
"-d",
|
||||||
|
database,
|
||||||
|
"-f",
|
||||||
|
dump_path,
|
||||||
|
)
|
||||||
|
result["backup_seconds"] = time.perf_counter() - started
|
||||||
|
result["backup_bytes"] = int(
|
||||||
|
compose("exec", "-T", "postgres", "stat", "-c", "%s", dump_path)
|
||||||
|
)
|
||||||
|
after = inside("after", database)
|
||||||
|
started = time.perf_counter()
|
||||||
|
compose(
|
||||||
|
"exec", "-T", "postgres", "createdb", "-U", "messenger", restored
|
||||||
|
)
|
||||||
|
compose(
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"postgres",
|
||||||
|
"pg_restore",
|
||||||
|
"-U",
|
||||||
|
"messenger",
|
||||||
|
"--exit-on-error",
|
||||||
|
"-d",
|
||||||
|
restored,
|
||||||
|
dump_path,
|
||||||
|
)
|
||||||
|
result["restore_seconds"] = time.perf_counter() - started
|
||||||
|
checked = inside("snapshot", restored)
|
||||||
|
expected = result["snapshot_before_backup"]
|
||||||
|
assert expected == checked
|
||||||
|
assert after["count"] == expected["counts"]["messages"] + 1
|
||||||
|
result["restore_verification"] = {
|
||||||
|
"snapshot_equal": True,
|
||||||
|
"selected_messages_checked": len(checked["selected_messages"]),
|
||||||
|
"restored": checked,
|
||||||
|
"post_snapshot_commits_not_restored": 1,
|
||||||
|
"source_count_after_backup": after["count"],
|
||||||
|
"post_snapshot_message": after["post_backup_message"],
|
||||||
|
}
|
||||||
|
result["postgres_cgroup_memory_current"] = int(
|
||||||
|
compose(
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"postgres",
|
||||||
|
"cat",
|
||||||
|
"/sys/fs/cgroup/memory.current",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result["postgres_cgroup_memory_peak"] = int(
|
||||||
|
compose(
|
||||||
|
"exec", "-T", "postgres", "cat", "/sys/fs/cgroup/memory.peak"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result["result"] = "PASS"
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(
|
||||||
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"PASS:",
|
||||||
|
result["dataset"]["messages"],
|
||||||
|
"messages;",
|
||||||
|
len(result["reads"]),
|
||||||
|
"EXPLAIN plans; backup/restore verified",
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
# Удаляются только две БД с UUID именно этого запуска.
|
||||||
|
for selected in [database, restored]:
|
||||||
|
inside("cleanup", selected)
|
||||||
|
compose("exec", "-T", "postgres", "rm", "-f", dump_path)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--output", type=Path)
|
||||||
|
parser.add_argument(
|
||||||
|
"--phase", choices=["plan", "after", "snapshot", "cleanup"]
|
||||||
|
)
|
||||||
|
parser.add_argument("--database")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.phase:
|
||||||
|
function = {
|
||||||
|
"plan": inside_plan,
|
||||||
|
"after": inside_after,
|
||||||
|
"snapshot": inside_snapshot,
|
||||||
|
"cleanup": lambda name: inside_cleanup([name]),
|
||||||
|
}[args.phase]
|
||||||
|
print(json.dumps(function(args.database), ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
outside(args.output)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def create(path, value):
|
||||||
|
try:
|
||||||
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||||
|
except FileExistsError:
|
||||||
|
return
|
||||||
|
with os.fdopen(fd, "w") as stream:
|
||||||
|
stream.write(value + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
directory = ROOT / "secrets"
|
||||||
|
directory.mkdir(mode=0o700, exist_ok=True)
|
||||||
|
pg = directory / "postgres_password"
|
||||||
|
cache = directory / "cache_password"
|
||||||
|
create(pg, secrets.token_urlsafe(32))
|
||||||
|
create(cache, secrets.token_urlsafe(32))
|
||||||
|
create(
|
||||||
|
directory / "database_url",
|
||||||
|
"postgresql://messenger:"
|
||||||
|
+ pg.read_text().strip()
|
||||||
|
+ "@postgres:5432/messenger",
|
||||||
|
)
|
||||||
|
create(
|
||||||
|
directory / "cache_url",
|
||||||
|
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
|
||||||
|
)
|
||||||
|
create(
|
||||||
|
directory / "valkey.conf",
|
||||||
|
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
|
||||||
|
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
|
||||||
|
+ cache.read_text().strip(),
|
||||||
|
)
|
||||||
|
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
|
||||||
|
print("Local configuration ready; existing files preserved")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+5
@@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
docker compose restart api postgres cache >/dev/null
|
||||||
|
docker compose up -d --no-recreate --wait >/dev/null
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8082")
|
||||||
|
|
||||||
|
|
||||||
|
def command(args, *, input=None):
|
||||||
|
result = subprocess.run(
|
||||||
|
args, cwd=ROOT, input=input, capture_output=True, text=True
|
||||||
|
)
|
||||||
|
if result.returncode:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Command failed: "
|
||||||
|
+ args[0]
|
||||||
|
+ "; "
|
||||||
|
+ result.stderr.replace(str(ROOT), "<repository>")
|
||||||
|
)
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def compose(*args):
|
||||||
|
return command(["docker", "compose", *args])
|
||||||
|
|
||||||
|
|
||||||
|
def sql(query, *, database="messenger"):
|
||||||
|
return compose(
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"postgres",
|
||||||
|
"psql",
|
||||||
|
"-U",
|
||||||
|
"messenger",
|
||||||
|
"-d",
|
||||||
|
database,
|
||||||
|
"-A",
|
||||||
|
"-t",
|
||||||
|
"-v",
|
||||||
|
"ON_ERROR_STOP=1",
|
||||||
|
"-c",
|
||||||
|
query,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def cache(command_name, *args):
|
||||||
|
script = (
|
||||||
|
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
|
||||||
|
'exec valkey-cli "$@"'
|
||||||
|
)
|
||||||
|
return compose(
|
||||||
|
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def ready(base=BASE, *, timeout=30):
|
||||||
|
end = time.monotonic() + timeout
|
||||||
|
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
|
||||||
|
while time.monotonic() < end:
|
||||||
|
try:
|
||||||
|
if client.get("/health/ready").status_code == 200:
|
||||||
|
return
|
||||||
|
except httpx.HTTPError:
|
||||||
|
pass
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise TimeoutError("API readiness")
|
||||||
|
|
||||||
|
|
||||||
|
def actor(client):
|
||||||
|
username = "verify_" + uuid4().hex[:20]
|
||||||
|
password = "Synthetic-" + uuid4().hex + ":🌍"
|
||||||
|
result = client.post(
|
||||||
|
"/api/v1/users",
|
||||||
|
json={
|
||||||
|
"username": username,
|
||||||
|
"display_name": "Synthetic",
|
||||||
|
"password": password,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.status_code == 201
|
||||||
|
user = result.json()
|
||||||
|
session = client.post(
|
||||||
|
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
|
||||||
|
)
|
||||||
|
assert session.status_code == 201
|
||||||
|
return user, password, session.json()
|
||||||
|
|
||||||
|
|
||||||
|
def auth(session):
|
||||||
|
return {"Authorization": "Bearer " + session["token"]}
|
||||||
|
|
||||||
|
|
||||||
|
def extra_api(*, ttl=3600, pool_size=1):
|
||||||
|
name = "mtusi-lab2-test-" + uuid4().hex[:10]
|
||||||
|
compose(
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--no-deps",
|
||||||
|
"--name",
|
||||||
|
name,
|
||||||
|
"-e",
|
||||||
|
"SESSION_TTL_SECONDS=" + str(ttl),
|
||||||
|
"-e",
|
||||||
|
"DB_POOL_SIZE=" + str(pool_size),
|
||||||
|
"-p",
|
||||||
|
"127.0.0.1::8080",
|
||||||
|
"api",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
info = json.loads(command(["docker", "inspect", name]))[0]
|
||||||
|
base = (
|
||||||
|
"http://127.0.0.1:"
|
||||||
|
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
|
||||||
|
)
|
||||||
|
ready(base)
|
||||||
|
return name, base
|
||||||
|
except Exception:
|
||||||
|
command(["docker", "rm", "-f", name])
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def instances():
|
||||||
|
data = {}
|
||||||
|
for role in ["api", "postgres", "cache"]:
|
||||||
|
cid = compose("ps", "-q", role)
|
||||||
|
info = json.loads(command(["docker", "inspect", cid]))[0]
|
||||||
|
data[role] = {
|
||||||
|
"container": cid[:12],
|
||||||
|
"pid": info["State"]["Pid"],
|
||||||
|
"started_at": info["State"]["StartedAt"],
|
||||||
|
}
|
||||||
|
return data
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
import argparse
|
||||||
|
import atexit
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
PROGRESS = {}
|
||||||
|
|
||||||
|
|
||||||
|
def save_incomplete():
|
||||||
|
receipt = PROGRESS.get("receipt")
|
||||||
|
if receipt and receipt.get("result") != "PASS":
|
||||||
|
target = PROGRESS["output"].with_name(
|
||||||
|
PROGRESS["output"].stem + "-failed.json"
|
||||||
|
)
|
||||||
|
if target.exists():
|
||||||
|
target = target.with_name(
|
||||||
|
target.stem + "-" + uuid4().hex[:8] + target.suffix
|
||||||
|
)
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{**receipt, "result": "INCOMPLETE"},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
+ "\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
atexit.register(save_incomplete)
|
||||||
|
|
||||||
|
|
||||||
|
def run(args, *, cwd, env, expected=0):
|
||||||
|
result = subprocess.run(
|
||||||
|
args, cwd=cwd, env=env, capture_output=True, text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
def redact(text):
|
||||||
|
return (
|
||||||
|
text.replace(str(cwd), "<clean-clone>")
|
||||||
|
.replace(str(ROOT), "<repository>")
|
||||||
|
.replace(str(Path.home()), "<home>")
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.returncode != expected:
|
||||||
|
if "receipt" in PROGRESS:
|
||||||
|
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
|
||||||
|
raise RuntimeError(redact(result.stdout + result.stderr))
|
||||||
|
return redact(result.stdout.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--output", default="evidence/clean-clone.json")
|
||||||
|
args = parser.parse_args()
|
||||||
|
output = ROOT / args.output
|
||||||
|
if output.exists():
|
||||||
|
raise SystemExit("Choose new output path")
|
||||||
|
assert not subprocess.check_output(
|
||||||
|
["git", "status", "--porcelain"], cwd=ROOT
|
||||||
|
)
|
||||||
|
sha = subprocess.check_output(
|
||||||
|
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
||||||
|
).strip()
|
||||||
|
identity = uuid4().hex[:10]
|
||||||
|
project = "mtusi-lab2-clean-" + identity
|
||||||
|
with socket.socket() as reservation:
|
||||||
|
reservation.bind(("127.0.0.1", 0))
|
||||||
|
public_port = str(reservation.getsockname()[1])
|
||||||
|
env = {
|
||||||
|
**os.environ,
|
||||||
|
"COMPOSE_PROJECT_NAME": project,
|
||||||
|
"API_IMAGE": "mtusi-messenger:lab2-clean-" + identity,
|
||||||
|
"PUBLIC_PORT": public_port,
|
||||||
|
"VCS_REF": sha,
|
||||||
|
"NO_PROXY": "localhost,127.0.0.1",
|
||||||
|
"no_proxy": "localhost,127.0.0.1",
|
||||||
|
}
|
||||||
|
receipt = {
|
||||||
|
"date": datetime.now(UTC).isoformat(),
|
||||||
|
"source_commit": sha,
|
||||||
|
"fresh_secrets_environment_and_volumes": True,
|
||||||
|
"command": "python scripts/verify_clean.py",
|
||||||
|
"fixed_host_port_for_restart": int(public_port),
|
||||||
|
}
|
||||||
|
PROGRESS.update(receipt=receipt, output=output)
|
||||||
|
with tempfile.TemporaryDirectory(prefix="mtusi-lab2-clean-") as temporary:
|
||||||
|
clone = Path(temporary) / "lab2"
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"git",
|
||||||
|
"clone",
|
||||||
|
"--no-hardlinks",
|
||||||
|
"--branch",
|
||||||
|
"solution/bvt2301-maslov",
|
||||||
|
str(ROOT),
|
||||||
|
str(clone),
|
||||||
|
],
|
||||||
|
cwd=ROOT,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
for name in [".env", ".venv", "secrets"]:
|
||||||
|
assert not (clone / name).exists()
|
||||||
|
run(["uv", "sync", "--locked"], cwd=clone, env=env)
|
||||||
|
receipt["static_checks"] = run(
|
||||||
|
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
run(["make", "bootstrap"], cwd=clone, env=env)
|
||||||
|
try:
|
||||||
|
run(
|
||||||
|
["docker", "compose", "up", "-d", "--build", "--wait"],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
api = run(
|
||||||
|
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
|
||||||
|
)
|
||||||
|
info = json.loads(
|
||||||
|
run(["docker", "inspect", api], cwd=clone, env=env)
|
||||||
|
)[0]
|
||||||
|
endpoint = info["NetworkSettings"]["Ports"]["8080/tcp"][0]
|
||||||
|
assert endpoint["HostIp"] == "127.0.0.1"
|
||||||
|
env["BASE_URL"] = "http://127.0.0.1:" + endpoint["HostPort"]
|
||||||
|
for role, port in [
|
||||||
|
("postgres", "5432/tcp"),
|
||||||
|
("cache", "6379/tcp"),
|
||||||
|
]:
|
||||||
|
cid = run(
|
||||||
|
["docker", "compose", "ps", "-q", role], cwd=clone, env=env
|
||||||
|
)
|
||||||
|
state = json.loads(
|
||||||
|
run(["docker", "inspect", cid], cwd=clone, env=env)
|
||||||
|
)[0]
|
||||||
|
assert not state["NetworkSettings"]["Ports"].get(port)
|
||||||
|
receipt["internal_store_ports_not_published"] = True
|
||||||
|
receipt["smoke"] = run(
|
||||||
|
["make", "test", "BASE_URL=" + env["BASE_URL"]],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
receipt["http_tests"] = run(
|
||||||
|
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
|
||||||
|
)
|
||||||
|
receipt["public_persistence"] = run(
|
||||||
|
[
|
||||||
|
"make",
|
||||||
|
"test-persistence",
|
||||||
|
"ACTION_SCRIPT=scripts/restart.sh",
|
||||||
|
"BASE_URL=" + env["BASE_URL"],
|
||||||
|
],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
for module, filename in [
|
||||||
|
("verify_lifecycle", "clean-lifecycle.json"),
|
||||||
|
("verify_pool", "clean-pool.json"),
|
||||||
|
]:
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
".venv/bin/python",
|
||||||
|
"-m",
|
||||||
|
"scripts." + module,
|
||||||
|
"--output",
|
||||||
|
"evidence/" + filename,
|
||||||
|
],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
receipt[module] = json.loads(
|
||||||
|
(clone / "evidence" / filename).read_text()
|
||||||
|
)
|
||||||
|
# Подменяется только временная read-only копия миграции.
|
||||||
|
altered = clone / ".work/altered-migrations"
|
||||||
|
altered.mkdir(parents=True)
|
||||||
|
migration = clone / "migrations/001_core.sql"
|
||||||
|
(altered / migration.name).write_bytes(
|
||||||
|
migration.read_bytes() + b"\n-- synthetic checksum mismatch\n"
|
||||||
|
)
|
||||||
|
attempt = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"compose",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--no-deps",
|
||||||
|
"-v",
|
||||||
|
str(altered) + ":/app/migrations:ro",
|
||||||
|
"migrate",
|
||||||
|
],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert attempt.returncode != 0
|
||||||
|
assert "Applied migration checksum mismatch" in attempt.stderr
|
||||||
|
receipt["applied_migration_checksum_guard"] = "PASS"
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
".venv/bin/python",
|
||||||
|
"-m",
|
||||||
|
"research.query_plan",
|
||||||
|
"--output",
|
||||||
|
"evidence/clean-query-plan.json",
|
||||||
|
],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
research = json.loads(
|
||||||
|
(clone / "evidence/clean-query-plan.json").read_text()
|
||||||
|
)
|
||||||
|
receipt["research_repeated"] = {
|
||||||
|
"result": research["result"],
|
||||||
|
"messages": research["dataset"]["messages"],
|
||||||
|
"plans": len(research["reads"]),
|
||||||
|
"restored_snapshot_equal": research["restore_verification"][
|
||||||
|
"snapshot_equal"
|
||||||
|
],
|
||||||
|
}
|
||||||
|
receipt["fsck"] = run(
|
||||||
|
["git", "fsck", "--full"], cwd=clone, env=env
|
||||||
|
)
|
||||||
|
receipt["checks_completed"] = True
|
||||||
|
finally:
|
||||||
|
assert project.startswith("mtusi-lab2-clean-")
|
||||||
|
run(
|
||||||
|
["docker", "compose", "down", "--volumes", "--remove-orphans"],
|
||||||
|
cwd=clone,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
receipt["result"] = "PASS"
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
|
||||||
|
print(json.dumps(receipt, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,311 @@
|
|||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from scripts.stand import (
|
||||||
|
BASE,
|
||||||
|
ROOT,
|
||||||
|
actor,
|
||||||
|
auth,
|
||||||
|
cache,
|
||||||
|
command,
|
||||||
|
compose,
|
||||||
|
extra_api,
|
||||||
|
instances,
|
||||||
|
ready,
|
||||||
|
sql,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--output", required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
output = Path(args.output)
|
||||||
|
if output.exists():
|
||||||
|
raise SystemExit("Choose new output path")
|
||||||
|
receipt = {
|
||||||
|
"date": datetime.now(UTC).isoformat(),
|
||||||
|
"commit": command(["git", "rev-parse", "HEAD"]),
|
||||||
|
"command": "python -m scripts.verify_lifecycle",
|
||||||
|
}
|
||||||
|
ready()
|
||||||
|
source_hashes = {
|
||||||
|
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||||
|
for p in (ROOT / "src").glob("*.py")
|
||||||
|
}
|
||||||
|
installed_hashes = json.loads(
|
||||||
|
compose(
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"api",
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
"import hashlib,json;from pathlib import Path;"
|
||||||
|
"print(json.dumps({p.name:"
|
||||||
|
"hashlib.sha256(p.read_bytes()).hexdigest() "
|
||||||
|
"for p in Path('/app/src').glob('*.py')}))",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert source_hashes == installed_hashes
|
||||||
|
receipt["runtime_source_hashes_match"] = True
|
||||||
|
api_info = json.loads(
|
||||||
|
command(["docker", "inspect", compose("ps", "-q", "api")])
|
||||||
|
)[0]
|
||||||
|
receipt["image_revision"] = api_info["Config"]["Labels"][
|
||||||
|
"org.opencontainers.image.revision"
|
||||||
|
]
|
||||||
|
with httpx.Client(base_url=BASE, trust_env=False, timeout=15) as client:
|
||||||
|
user, password, session = actor(client)
|
||||||
|
headers = auth(session)
|
||||||
|
expiry_query = (
|
||||||
|
"SELECT expires_at::text FROM sessions WHERE user_id='"
|
||||||
|
+ user["id"]
|
||||||
|
+ "'"
|
||||||
|
)
|
||||||
|
expires_before = sql(expiry_query)
|
||||||
|
chat = client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers,
|
||||||
|
json={"title": "Lifecycle", "member_ids": [user["id"]]},
|
||||||
|
).json()
|
||||||
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||||
|
message = client.post(
|
||||||
|
path, headers=headers, json={"text": "Synthetic retained message"}
|
||||||
|
)
|
||||||
|
assert message.status_code == 201
|
||||||
|
message_id = message.json()["id"]
|
||||||
|
before = instances()
|
||||||
|
started = time.perf_counter()
|
||||||
|
command(["sh", "scripts/restart.sh"])
|
||||||
|
ready()
|
||||||
|
after = instances()
|
||||||
|
assert sql(expiry_query) == expires_before
|
||||||
|
for role in before:
|
||||||
|
assert before[role]["started_at"] != after[role]["started_at"]
|
||||||
|
assert (
|
||||||
|
client.get(path, headers=headers).json()["items"][0]["id"]
|
||||||
|
== message_id
|
||||||
|
)
|
||||||
|
receipt["persistence"] = {
|
||||||
|
"absolute_expiry_unchanged": True,
|
||||||
|
"same_token": True,
|
||||||
|
"same_message": True,
|
||||||
|
"before": before,
|
||||||
|
"after": after,
|
||||||
|
"seconds": time.perf_counter() - started,
|
||||||
|
}
|
||||||
|
migration_count = sql("SELECT count(*) FROM schema_migrations")
|
||||||
|
compose("run", "--rm", "--no-deps", "migrate")
|
||||||
|
assert sql("SELECT count(*) FROM schema_migrations") == migration_count
|
||||||
|
receipt["migration_repeat"] = "PASS"
|
||||||
|
files_before = {
|
||||||
|
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||||
|
for p in (ROOT / "secrets").iterdir()
|
||||||
|
}
|
||||||
|
command(["python3", "scripts/bootstrap.py"])
|
||||||
|
assert files_before == {
|
||||||
|
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||||
|
for p in (ROOT / "secrets").iterdir()
|
||||||
|
}
|
||||||
|
receipt["bootstrap_preserves_secrets"] = True
|
||||||
|
dump = compose(
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"postgres",
|
||||||
|
"pg_dump",
|
||||||
|
"-U",
|
||||||
|
"messenger",
|
||||||
|
"--data-only",
|
||||||
|
"messenger",
|
||||||
|
)
|
||||||
|
assert password not in dump and session["token"] not in dump
|
||||||
|
del dump
|
||||||
|
privacy = json.loads(
|
||||||
|
sql(
|
||||||
|
"SELECT json_build_object('argon2id',"
|
||||||
|
"password_hash LIKE '$argon2id$%','digest_bytes',"
|
||||||
|
"(SELECT min(octet_length(token_digest)) FROM sessions "
|
||||||
|
"WHERE user_id=users.id)) FROM users WHERE id='"
|
||||||
|
+ user["id"]
|
||||||
|
+ "'"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert privacy == {"argon2id": True, "digest_bytes": 32}
|
||||||
|
receipt["credential_storage"] = {
|
||||||
|
**privacy,
|
||||||
|
"raw_password_and_token_absent_in_dump": True,
|
||||||
|
}
|
||||||
|
cache("FLUSHDB")
|
||||||
|
assert (
|
||||||
|
client.get("/api/v1/users/me", headers=headers).status_code == 200
|
||||||
|
)
|
||||||
|
info_before = cache("INFO", "stats")
|
||||||
|
for _ in range(20):
|
||||||
|
assert (
|
||||||
|
client.get("/api/v1/users/me", headers=headers).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
info_after = cache("INFO", "stats")
|
||||||
|
|
||||||
|
def hits(value):
|
||||||
|
return int(
|
||||||
|
next(
|
||||||
|
line.split(":")[1]
|
||||||
|
for line in value.splitlines()
|
||||||
|
if line.startswith("keyspace_hits:")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert hits(info_after) - hits(info_before) >= 20
|
||||||
|
receipt["cache_hits"] = hits(info_after) - hits(info_before)
|
||||||
|
compose("stop", "cache")
|
||||||
|
try:
|
||||||
|
started = time.perf_counter()
|
||||||
|
assert (
|
||||||
|
client.get("/api/v1/users/me", headers=headers).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
receipt["cache_stopped"] = {
|
||||||
|
"status": 200,
|
||||||
|
"seconds": time.perf_counter() - started,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
compose("up", "-d", "--no-recreate", "--wait", "cache")
|
||||||
|
cache("FLUSHDB")
|
||||||
|
assert (
|
||||||
|
client.get(path, headers=headers).json()["items"][0]["id"]
|
||||||
|
== message_id
|
||||||
|
)
|
||||||
|
receipt["cache_flush_preserves_truth"] = True
|
||||||
|
for mode in ["pause", "stop"]:
|
||||||
|
# Дожидаемся окончания circuit backoff и доказываем cache hit
|
||||||
|
# непосредственно перед отказом источника истины.
|
||||||
|
for _ in range(30):
|
||||||
|
assert (
|
||||||
|
client.get("/api/v1/users/me", headers=headers).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
if cache("EXISTS", "user:" + user["id"]) == "1":
|
||||||
|
break
|
||||||
|
time.sleep(0.25)
|
||||||
|
else:
|
||||||
|
raise AssertionError("User cache not primed before PG fault")
|
||||||
|
compose(mode, "postgres")
|
||||||
|
try:
|
||||||
|
started = time.perf_counter()
|
||||||
|
result = client.get("/api/v1/users/me", headers=headers)
|
||||||
|
elapsed = time.perf_counter() - started
|
||||||
|
assert result.status_code == 503 and elapsed < 6
|
||||||
|
assert result.json()["error"]["code"] == "unavailable"
|
||||||
|
assert (
|
||||||
|
result.headers["x-request-id"]
|
||||||
|
== result.json()["error"]["request_id"]
|
||||||
|
)
|
||||||
|
assert client.get("/health/live").status_code == 200
|
||||||
|
receipt["postgres_" + mode] = {
|
||||||
|
"cache_card_present_before_failure": True,
|
||||||
|
"status": result.status_code,
|
||||||
|
"seconds": elapsed,
|
||||||
|
"cached_identity_not_trusted": True,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
if mode == "pause":
|
||||||
|
compose("unpause", "postgres")
|
||||||
|
else:
|
||||||
|
compose("up", "-d", "--no-recreate", "--wait", "postgres")
|
||||||
|
ready(timeout=45)
|
||||||
|
assert (
|
||||||
|
client.get(path, headers=headers).json()["items"][0]["id"]
|
||||||
|
== message_id
|
||||||
|
)
|
||||||
|
# Отказ посередине вставки участников откатывает весь чат.
|
||||||
|
outsider, _, _ = actor(client)
|
||||||
|
title = "rollback_" + uuid4().hex
|
||||||
|
function = "rollback_" + uuid4().hex[:8]
|
||||||
|
sql(
|
||||||
|
"CREATE FUNCTION "
|
||||||
|
+ function
|
||||||
|
+ "() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN "
|
||||||
|
+ "IF NEW.user_id='"
|
||||||
|
+ outsider["id"]
|
||||||
|
+ "'::uuid THEN RAISE EXCEPTION 'synthetic member failure'; "
|
||||||
|
+ "END IF; RETURN NEW; END $$; CREATE TRIGGER "
|
||||||
|
+ function
|
||||||
|
+ " BEFORE INSERT ON members FOR EACH ROW EXECUTE FUNCTION "
|
||||||
|
+ function
|
||||||
|
+ "()"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"title": title,
|
||||||
|
"member_ids": [user["id"], outsider["id"]],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.status_code == 500
|
||||||
|
assert (
|
||||||
|
sql("SELECT count(*) FROM chats WHERE title='" + title + "'")
|
||||||
|
== "0"
|
||||||
|
)
|
||||||
|
receipt["chat_transaction_rollback"] = {
|
||||||
|
"injected_failure_status": 500,
|
||||||
|
"remaining_chat_rows": 0,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
sql(
|
||||||
|
"DROP TRIGGER "
|
||||||
|
+ function
|
||||||
|
+ " ON members; DROP FUNCTION "
|
||||||
|
+ function
|
||||||
|
+ "()"
|
||||||
|
)
|
||||||
|
# Короткий TTL — на отдельном API; обычный smoke сохраняет TTL3600.
|
||||||
|
name, base = extra_api(ttl=2)
|
||||||
|
try:
|
||||||
|
with httpx.Client(
|
||||||
|
base_url=base, trust_env=False, timeout=15
|
||||||
|
) as short:
|
||||||
|
_, _, expiring = actor(short)
|
||||||
|
assert (
|
||||||
|
short.get(
|
||||||
|
"/api/v1/users/me", headers=auth(expiring)
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
time.sleep(2.2)
|
||||||
|
assert (
|
||||||
|
short.get(
|
||||||
|
"/api/v1/users/me", headers=auth(expiring)
|
||||||
|
).status_code
|
||||||
|
== 401
|
||||||
|
)
|
||||||
|
receipt["expiry"] = {"ttl_seconds": 2, "status_after_expiry": 401}
|
||||||
|
finally:
|
||||||
|
command(["docker", "rm", "-f", name])
|
||||||
|
assert (
|
||||||
|
client.delete(
|
||||||
|
"/api/v1/auth/sessions/current", headers=headers
|
||||||
|
).status_code
|
||||||
|
== 204
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
client.get("/api/v1/users/me", headers=headers).status_code == 401
|
||||||
|
)
|
||||||
|
receipt["logout_after_cache_hit"] = "PASS"
|
||||||
|
receipt["result"] = "PASS"
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
|
||||||
|
print(json.dumps(receipt, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from scripts.stand import actor, auth, command, extra_api, sql
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--output", required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
output = Path(args.output)
|
||||||
|
if output.exists():
|
||||||
|
raise SystemExit("Choose new output path")
|
||||||
|
name, base = extra_api(pool_size=1)
|
||||||
|
function = "poolwait_" + uuid4().hex[:8]
|
||||||
|
trigger_created = False
|
||||||
|
try:
|
||||||
|
with httpx.Client(
|
||||||
|
base_url=base, trust_env=False, timeout=15
|
||||||
|
) as client:
|
||||||
|
user, _, session = actor(client)
|
||||||
|
headers = auth(session)
|
||||||
|
chat = client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers,
|
||||||
|
json={"title": "Pool test", "member_ids": [user["id"]]},
|
||||||
|
).json()
|
||||||
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||||
|
sql(
|
||||||
|
"CREATE FUNCTION "
|
||||||
|
+ function
|
||||||
|
+ "() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='"
|
||||||
|
+ chat["id"]
|
||||||
|
+ "'::uuid THEN PERFORM pg_sleep(4); END IF; RETURN NEW; END $$; CREATE TRIGGER "
|
||||||
|
+ function
|
||||||
|
+ " BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION "
|
||||||
|
+ function
|
||||||
|
+ "()"
|
||||||
|
)
|
||||||
|
trigger_created = True
|
||||||
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||||
|
blocking = pool.submit(
|
||||||
|
client.post,
|
||||||
|
path,
|
||||||
|
headers=headers,
|
||||||
|
json={"text": "Synthetic delayed insert"},
|
||||||
|
)
|
||||||
|
for _ in range(30):
|
||||||
|
active = sql(
|
||||||
|
"SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND query LIKE 'INSERT INTO messages%'"
|
||||||
|
)
|
||||||
|
if int(active) > 0:
|
||||||
|
break
|
||||||
|
time.sleep(0.05)
|
||||||
|
else:
|
||||||
|
raise AssertionError("Blocked INSERT not observed")
|
||||||
|
started = time.perf_counter()
|
||||||
|
second = client.get("/api/v1/users/me", headers=headers)
|
||||||
|
waited = time.perf_counter() - started
|
||||||
|
assert second.status_code == 503 and 0.8 <= waited < 3
|
||||||
|
first = blocking.result(timeout=6)
|
||||||
|
assert first.status_code == 503
|
||||||
|
sql(
|
||||||
|
"DROP TRIGGER "
|
||||||
|
+ function
|
||||||
|
+ " ON messages; DROP FUNCTION "
|
||||||
|
+ function
|
||||||
|
+ "()"
|
||||||
|
)
|
||||||
|
trigger_created = False
|
||||||
|
assert client.get(path, headers=headers).json()["items"] == []
|
||||||
|
assert (
|
||||||
|
client.post(
|
||||||
|
path, headers=headers, json={"text": "After recovery"}
|
||||||
|
).status_code
|
||||||
|
== 201
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"date": datetime.now(UTC).isoformat(),
|
||||||
|
"commit": command(["git", "rev-parse", "HEAD"]),
|
||||||
|
"command": "python -m scripts.verify_pool",
|
||||||
|
"pool_size": 1,
|
||||||
|
"blocked_insert_observed_in_pg_stat_activity": True,
|
||||||
|
"pool_wait_seconds": waited,
|
||||||
|
"pool_timeout_status": second.status_code,
|
||||||
|
"statement_timeout_status": first.status_code,
|
||||||
|
"failed_write_rolled_back": True,
|
||||||
|
"write_after_recovery": 201,
|
||||||
|
"result": "PASS",
|
||||||
|
}
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(
|
||||||
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||||
|
)
|
||||||
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
|
finally:
|
||||||
|
if trigger_created:
|
||||||
|
sql(
|
||||||
|
"DROP TRIGGER IF EXISTS "
|
||||||
|
+ function
|
||||||
|
+ " ON messages; DROP FUNCTION IF EXISTS "
|
||||||
|
+ function
|
||||||
|
+ "()"
|
||||||
|
)
|
||||||
|
command(["docker", "rm", "-f", name])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def early_stop(signum, _frame):
|
||||||
|
print(
|
||||||
|
json.dumps({"event": "stopped_before_startup", "signal": signum}),
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, early_stop)
|
||||||
|
signal.signal(signal.SIGINT, early_stop)
|
||||||
|
|
||||||
|
import uvicorn # noqa: E402
|
||||||
|
|
||||||
|
from src.logging_config import JsonFormatter # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
host = os.environ.get("APP_HOST", "127.0.0.1")
|
||||||
|
port = int(os.environ.get("APP_PORT", "8080"))
|
||||||
|
if not 1 <= port <= 65535:
|
||||||
|
raise ValueError("APP_PORT must be between 1 and 65535")
|
||||||
|
handler = logging.StreamHandler(sys.stdout)
|
||||||
|
handler.setFormatter(JsonFormatter())
|
||||||
|
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
|
||||||
|
uvicorn.run(
|
||||||
|
"src.api:app",
|
||||||
|
host=host,
|
||||||
|
port=port,
|
||||||
|
workers=1,
|
||||||
|
access_log=False,
|
||||||
|
timeout_graceful_shutdown=5,
|
||||||
|
log_config=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+298
@@ -0,0 +1,298 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from datetime import datetime
|
||||||
|
from functools import partial
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from anyio import to_thread
|
||||||
|
from fastapi import FastAPI, Request
|
||||||
|
from fastapi.encoders import jsonable_encoder
|
||||||
|
from pydantic import ValidationError
|
||||||
|
from starlette.exceptions import HTTPException
|
||||||
|
from starlette.responses import JSONResponse, Response
|
||||||
|
|
||||||
|
from src.cache import UserCache
|
||||||
|
from src.cursor import parse_uuid
|
||||||
|
from src.database import Database
|
||||||
|
from src.domain import timestamp
|
||||||
|
from src.errors import DomainError, invalid
|
||||||
|
from src.password import Passwords
|
||||||
|
from src.repository import PostgresRepository
|
||||||
|
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
|
||||||
|
from src.service import MessengerService
|
||||||
|
from src.settings import Settings
|
||||||
|
|
||||||
|
logger = logging.getLogger("messenger")
|
||||||
|
|
||||||
|
|
||||||
|
def response(data: object, *, status: int = 200) -> JSONResponse:
|
||||||
|
return JSONResponse(
|
||||||
|
jsonable_encoder(
|
||||||
|
data, custom_encoder={datetime: timestamp, UUID: str}
|
||||||
|
),
|
||||||
|
status_code=status,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def body(request: Request, model: type[Input]) -> Input:
|
||||||
|
try:
|
||||||
|
data = await request.body()
|
||||||
|
return model.model_validate(
|
||||||
|
json.loads(
|
||||||
|
data.decode("utf-8"),
|
||||||
|
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except ValueError, ValidationError, UnicodeError, RecursionError:
|
||||||
|
raise invalid() from None
|
||||||
|
|
||||||
|
|
||||||
|
def pagination(request: Request) -> tuple[int, str | None]:
|
||||||
|
limits = request.query_params.getlist("limit")
|
||||||
|
cursors = request.query_params.getlist("cursor")
|
||||||
|
if len(limits) > 1 or len(cursors) > 1:
|
||||||
|
raise invalid()
|
||||||
|
raw = limits[0] if limits else "50"
|
||||||
|
if not raw.isascii() or not raw.isdecimal():
|
||||||
|
raise invalid()
|
||||||
|
raw = raw.lstrip("0") or "0"
|
||||||
|
if len(raw) > 3:
|
||||||
|
raise invalid()
|
||||||
|
limit = int(raw)
|
||||||
|
if not 1 <= limit <= 100:
|
||||||
|
raise invalid()
|
||||||
|
return limit, cursors[0] if cursors else None
|
||||||
|
|
||||||
|
|
||||||
|
def service(request: Request) -> MessengerService:
|
||||||
|
return request.app.state.service
|
||||||
|
|
||||||
|
|
||||||
|
async def run_io(function, **kwargs):
|
||||||
|
return await to_thread.run_sync(partial(function, **kwargs))
|
||||||
|
|
||||||
|
|
||||||
|
async def actor(request: Request):
|
||||||
|
return await run_io(
|
||||||
|
service(request).identify, value=request.headers.get("authorization")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||||
|
settings = Settings.load()
|
||||||
|
database = Database(url=settings.database_url, size=settings.pool_size)
|
||||||
|
cache = UserCache(url=settings.cache_url)
|
||||||
|
try:
|
||||||
|
await run_io(database.open)
|
||||||
|
passwords = await run_io(Passwords)
|
||||||
|
app.state.database = database
|
||||||
|
app.state.service = MessengerService(
|
||||||
|
repository=PostgresRepository(database=database, cache=cache),
|
||||||
|
passwords=passwords,
|
||||||
|
session_ttl=settings.session_ttl,
|
||||||
|
)
|
||||||
|
app.state.ready = True
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
app.state.ready = False
|
||||||
|
await run_io(cache.close)
|
||||||
|
await run_io(database.close)
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(
|
||||||
|
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.middleware("http")
|
||||||
|
async def request_context(request: Request, call_next):
|
||||||
|
request_id = str(uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
started = time.perf_counter()
|
||||||
|
try:
|
||||||
|
result = await call_next(request)
|
||||||
|
except Exception:
|
||||||
|
logger.error(
|
||||||
|
json.dumps({"event": "internal_error", "request_id": request_id})
|
||||||
|
)
|
||||||
|
result = response(
|
||||||
|
{
|
||||||
|
"error": {
|
||||||
|
"code": "internal_error",
|
||||||
|
"message": "Internal error",
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
status=500,
|
||||||
|
)
|
||||||
|
result.headers["X-Request-Id"] = request_id
|
||||||
|
route = request.scope.get("route")
|
||||||
|
logger.info(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"event": "request",
|
||||||
|
"request_id": request_id,
|
||||||
|
"method": request.method,
|
||||||
|
"route": getattr(route, "path", "unmatched"),
|
||||||
|
"status": result.status_code,
|
||||||
|
"elapsed_ms": round((time.perf_counter() - started) * 1000, 3),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(DomainError)
|
||||||
|
async def domain_error(request: Request, error: DomainError):
|
||||||
|
result = response(
|
||||||
|
{
|
||||||
|
"error": {
|
||||||
|
"code": error.code,
|
||||||
|
"message": error.message,
|
||||||
|
"request_id": request.state.request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
status=error.status,
|
||||||
|
)
|
||||||
|
result.headers.update(error.headers)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(HTTPException)
|
||||||
|
async def http_error(request: Request, error: HTTPException):
|
||||||
|
code = "not_found" if error.status_code == 404 else "invalid_request"
|
||||||
|
return response(
|
||||||
|
{
|
||||||
|
"error": {
|
||||||
|
"code": code,
|
||||||
|
"message": "Resource not found"
|
||||||
|
if error.status_code == 404
|
||||||
|
else "Invalid request",
|
||||||
|
"request_id": request.state.request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
status=error.status_code,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health/live")
|
||||||
|
async def live():
|
||||||
|
return response({"status": "ok"})
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health/ready")
|
||||||
|
async def ready():
|
||||||
|
if not getattr(app.state, "ready", False):
|
||||||
|
raise DomainError(503, "unavailable", "Not ready")
|
||||||
|
await run_io(app.state.database.ping)
|
||||||
|
return response({"status": "ok"})
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/users")
|
||||||
|
async def register(request: Request):
|
||||||
|
data = await body(request, CreateUser)
|
||||||
|
return response(
|
||||||
|
await run_io(service(request).register, **data.model_dump()),
|
||||||
|
status=201,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/users/me")
|
||||||
|
async def current(request: Request):
|
||||||
|
return response(await actor(request))
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/chats")
|
||||||
|
async def create_chat(request: Request):
|
||||||
|
user = await actor(request)
|
||||||
|
data = await body(request, CreateChat)
|
||||||
|
return response(
|
||||||
|
await run_io(
|
||||||
|
service(request).create_chat, actor_id=user.id, **data.model_dump()
|
||||||
|
),
|
||||||
|
status=201,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats")
|
||||||
|
async def list_chats(request: Request):
|
||||||
|
user = await actor(request)
|
||||||
|
limit, cursor = pagination(request)
|
||||||
|
return response(
|
||||||
|
await run_io(
|
||||||
|
service(request).chats,
|
||||||
|
actor_id=user.id,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats/{chat_id}")
|
||||||
|
async def get_chat(request: Request, chat_id: str):
|
||||||
|
user = await actor(request)
|
||||||
|
return response(
|
||||||
|
await run_io(
|
||||||
|
service(request).get_chat,
|
||||||
|
actor_id=user.id,
|
||||||
|
chat_id=parse_uuid(chat_id),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/chats/{chat_id}/messages")
|
||||||
|
async def send(request: Request, chat_id: str):
|
||||||
|
user = await actor(request)
|
||||||
|
cid = parse_uuid(chat_id)
|
||||||
|
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
|
||||||
|
data = await body(request, CreateMessage)
|
||||||
|
return response(
|
||||||
|
await run_io(
|
||||||
|
service(request).send,
|
||||||
|
actor_id=user.id,
|
||||||
|
chat_id=cid,
|
||||||
|
**data.model_dump(),
|
||||||
|
),
|
||||||
|
status=201,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||||
|
async def history(request: Request, chat_id: str):
|
||||||
|
user = await actor(request)
|
||||||
|
cid = parse_uuid(chat_id)
|
||||||
|
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
|
||||||
|
limit, cursor = pagination(request)
|
||||||
|
return response(
|
||||||
|
await run_io(
|
||||||
|
service(request).history,
|
||||||
|
actor_id=user.id,
|
||||||
|
chat_id=cid,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/auth/sessions")
|
||||||
|
async def login(request: Request):
|
||||||
|
return response(
|
||||||
|
await run_io(
|
||||||
|
service(request).login,
|
||||||
|
authorization=request.headers.get("authorization"),
|
||||||
|
),
|
||||||
|
status=201,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/api/v1/auth/sessions/current")
|
||||||
|
async def logout(request: Request):
|
||||||
|
await run_io(
|
||||||
|
service(request).logout,
|
||||||
|
authorization=request.headers.get("authorization"),
|
||||||
|
)
|
||||||
|
return Response(status_code=204)
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import json
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import redis
|
||||||
|
from redis.backoff import NoBackoff
|
||||||
|
from redis.retry import Retry
|
||||||
|
|
||||||
|
from src.domain import User, timestamp
|
||||||
|
|
||||||
|
|
||||||
|
class UserCache:
|
||||||
|
def __init__(self, *, url: str):
|
||||||
|
self.client = redis.Redis.from_url(
|
||||||
|
url,
|
||||||
|
socket_connect_timeout=0.15,
|
||||||
|
socket_timeout=0.15,
|
||||||
|
max_connections=16,
|
||||||
|
retry=Retry(NoBackoff(), 0),
|
||||||
|
decode_responses=True,
|
||||||
|
)
|
||||||
|
self.disabled_until = 0.0
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.client.close()
|
||||||
|
|
||||||
|
def _command(self, function, *args, **kwargs):
|
||||||
|
if time.monotonic() < self.disabled_until:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return function(*args, **kwargs)
|
||||||
|
except redis.RedisError, OSError:
|
||||||
|
with self.lock:
|
||||||
|
self.disabled_until = time.monotonic() + 5
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get(self, *, user_id: UUID) -> User | None:
|
||||||
|
raw = self._command(self.client.get, "user:" + str(user_id))
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
row = json.loads(raw)
|
||||||
|
if UUID(row["id"]) != user_id:
|
||||||
|
return None
|
||||||
|
return User(
|
||||||
|
user_id,
|
||||||
|
row["username"],
|
||||||
|
row["display_name"],
|
||||||
|
datetime.fromisoformat(row["created_at"]),
|
||||||
|
)
|
||||||
|
except ValueError, TypeError, KeyError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def put(self, *, user: User):
|
||||||
|
raw = json.dumps(
|
||||||
|
{
|
||||||
|
"id": str(user.id),
|
||||||
|
"username": user.username,
|
||||||
|
"display_name": user.display_name,
|
||||||
|
"created_at": timestamp(user.created_at),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from src.domain import Key, timestamp
|
||||||
|
from src.errors import invalid
|
||||||
|
|
||||||
|
|
||||||
|
def parse_uuid(value: str) -> UUID:
|
||||||
|
if not isinstance(value, str) or not re.fullmatch(
|
||||||
|
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
|
||||||
|
):
|
||||||
|
raise invalid()
|
||||||
|
return UUID(value)
|
||||||
|
|
||||||
|
|
||||||
|
def encode_cursor(*, scope: str, key: Key) -> str:
|
||||||
|
payload = {
|
||||||
|
"v": 1,
|
||||||
|
"scope": scope,
|
||||||
|
"at": timestamp(key[0]),
|
||||||
|
"id": str(key[1]),
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
base64.urlsafe_b64encode(
|
||||||
|
json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
)
|
||||||
|
.decode()
|
||||||
|
.rstrip("=")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
if (
|
||||||
|
not value
|
||||||
|
or len(value) > 2048
|
||||||
|
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
|
||||||
|
):
|
||||||
|
raise ValueError
|
||||||
|
data = json.loads(
|
||||||
|
base64.b64decode(
|
||||||
|
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not isinstance(data, dict)
|
||||||
|
or type(data.get("v")) is not int
|
||||||
|
or data["v"] != 1
|
||||||
|
or data.get("scope") != scope
|
||||||
|
or not isinstance(data.get("at"), str)
|
||||||
|
):
|
||||||
|
raise ValueError
|
||||||
|
at = datetime.fromisoformat(data["at"])
|
||||||
|
if at.tzinfo is None or timestamp(at) != data["at"]:
|
||||||
|
raise ValueError
|
||||||
|
return at, parse_uuid(data["id"])
|
||||||
|
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
|
||||||
|
raise invalid() from None
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import time
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||||
|
|
||||||
|
from src.errors import DomainError
|
||||||
|
|
||||||
|
|
||||||
|
class DeadlineConnection(psycopg.Connection):
|
||||||
|
"""Клиентский срок действует и при замороженном PostgreSQL."""
|
||||||
|
|
||||||
|
def wait(self, gen, interval=0.1, timeout=None):
|
||||||
|
deadline = getattr(self, "request_deadline", None)
|
||||||
|
remaining = (
|
||||||
|
max(0.001, deadline - time.monotonic()) if deadline else 3.0
|
||||||
|
)
|
||||||
|
bound = min(remaining, timeout) if timeout is not None else remaining
|
||||||
|
try:
|
||||||
|
return super().wait(gen, interval=interval, timeout=bound)
|
||||||
|
except psycopg.OperationalError:
|
||||||
|
self.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
class Database:
|
||||||
|
def __init__(self, *, url: str, size: int):
|
||||||
|
self.pool = ConnectionPool(
|
||||||
|
url,
|
||||||
|
connection_class=DeadlineConnection,
|
||||||
|
kwargs={
|
||||||
|
"autocommit": True,
|
||||||
|
"row_factory": dict_row,
|
||||||
|
"connect_timeout": 2,
|
||||||
|
"options": (
|
||||||
|
"-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||||
|
"-c idle_in_transaction_session_timeout=6000"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
min_size=1,
|
||||||
|
max_size=size,
|
||||||
|
timeout=1,
|
||||||
|
open=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def open(self):
|
||||||
|
self.pool.open(wait=True, timeout=10)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.pool.close(timeout=2)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def connection(self):
|
||||||
|
try:
|
||||||
|
with self.pool.connection(timeout=1) as conn:
|
||||||
|
conn.request_deadline = time.monotonic() + 4
|
||||||
|
try:
|
||||||
|
yield conn
|
||||||
|
finally:
|
||||||
|
conn.request_deadline = None
|
||||||
|
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
|
||||||
|
raise DomainError(
|
||||||
|
503, "unavailable", "Database unavailable"
|
||||||
|
) from None
|
||||||
|
|
||||||
|
def ping(self):
|
||||||
|
with self.connection() as conn:
|
||||||
|
conn.execute("SELECT 1").fetchone()
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
def utcnow() -> datetime:
|
||||||
|
return datetime.now(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def timestamp(value: datetime) -> str:
|
||||||
|
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class User:
|
||||||
|
id: UUID
|
||||||
|
username: str
|
||||||
|
display_name: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Chat:
|
||||||
|
id: UUID
|
||||||
|
title: str
|
||||||
|
member_ids: tuple[UUID, ...]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Message:
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
sender_id: UUID
|
||||||
|
text: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
type Record = User | Chat | Message
|
||||||
|
type Key = tuple[datetime, UUID]
|
||||||
|
|
||||||
|
|
||||||
|
def order_key(record: Record) -> Key:
|
||||||
|
return record.created_at, record.id
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
class DomainError(Exception):
|
||||||
|
def __init__(
|
||||||
|
self, status: int, code: str, message: str, headers=None
|
||||||
|
) -> None:
|
||||||
|
self.status = status
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
self.headers = headers or {}
|
||||||
|
|
||||||
|
|
||||||
|
def invalid() -> DomainError:
|
||||||
|
return DomainError(400, "invalid_request", "Invalid request")
|
||||||
|
|
||||||
|
|
||||||
|
def not_found() -> DomainError:
|
||||||
|
return DomainError(404, "not_found", "Resource not found")
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
|
||||||
|
class JsonFormatter(logging.Formatter):
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
message = record.getMessage()
|
||||||
|
try:
|
||||||
|
event = json.loads(message)
|
||||||
|
if not isinstance(event, dict):
|
||||||
|
event = {"event": message}
|
||||||
|
except ValueError:
|
||||||
|
event = {"event": message}
|
||||||
|
event["timestamp"] = datetime.now(UTC).isoformat()
|
||||||
|
event["level"] = record.levelname.lower()
|
||||||
|
return json.dumps(event, ensure_ascii=False)
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
from src.settings import secret
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
with psycopg.connect(
|
||||||
|
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
|
||||||
|
) as conn:
|
||||||
|
conn.execute("SELECT pg_advisory_lock(2301002)")
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||||
|
"(name text PRIMARY KEY, checksum text NOT NULL, "
|
||||||
|
"applied_at timestamptz NOT NULL DEFAULT now())"
|
||||||
|
)
|
||||||
|
for path in sorted(Path("migrations").glob("*.sql")):
|
||||||
|
content = path.read_bytes()
|
||||||
|
checksum = hashlib.sha256(content).hexdigest()
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT checksum FROM schema_migrations WHERE name=%s",
|
||||||
|
(path.name,),
|
||||||
|
).fetchone()
|
||||||
|
if row:
|
||||||
|
if row[0] != checksum:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Applied migration checksum mismatch"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
with conn.transaction():
|
||||||
|
conn.execute(content.decode())
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO schema_migrations(name,checksum) "
|
||||||
|
"VALUES(%s,%s)",
|
||||||
|
(path.name, checksum),
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{"event": "migration_applied", "name": path.name}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
conn.execute("SELECT pg_advisory_unlock(2301002)")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import threading
|
||||||
|
|
||||||
|
from argon2 import PasswordHasher
|
||||||
|
from argon2.exceptions import InvalidHashError, VerificationError
|
||||||
|
from argon2.low_level import Type
|
||||||
|
|
||||||
|
from src.errors import DomainError
|
||||||
|
|
||||||
|
|
||||||
|
class Passwords:
|
||||||
|
def __init__(self):
|
||||||
|
self.hasher = PasswordHasher(
|
||||||
|
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
|
||||||
|
)
|
||||||
|
self.slots = threading.BoundedSemaphore(2)
|
||||||
|
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
|
||||||
|
|
||||||
|
def _run(self, function, *args):
|
||||||
|
if not self.slots.acquire(timeout=5):
|
||||||
|
raise DomainError(503, "unavailable", "Authentication busy")
|
||||||
|
try:
|
||||||
|
return function(*args)
|
||||||
|
finally:
|
||||||
|
self.slots.release()
|
||||||
|
|
||||||
|
def hash(self, *, password: str) -> str:
|
||||||
|
return self._run(self.hasher.hash, password)
|
||||||
|
|
||||||
|
def verify(self, *, password: str, encoded: str | None) -> bool:
|
||||||
|
try:
|
||||||
|
return (
|
||||||
|
self._run(self.hasher.verify, encoded or self.dummy, password)
|
||||||
|
and encoded is not None
|
||||||
|
)
|
||||||
|
except VerificationError, InvalidHashError:
|
||||||
|
return False
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from psycopg.errors import ForeignKeyViolation, UniqueViolation
|
||||||
|
|
||||||
|
from src.cache import UserCache
|
||||||
|
from src.database import Database
|
||||||
|
from src.domain import Chat, Key, Message, User
|
||||||
|
from src.errors import DomainError, invalid, not_found
|
||||||
|
|
||||||
|
USER_COLUMNS = "id, username, display_name, created_at"
|
||||||
|
|
||||||
|
|
||||||
|
def user_dto(row) -> User:
|
||||||
|
return User(
|
||||||
|
row["id"], row["username"], row["display_name"], row["created_at"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def chat_dto(row) -> Chat:
|
||||||
|
return Chat(
|
||||||
|
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def message_dto(row) -> Message:
|
||||||
|
return Message(
|
||||||
|
row["id"],
|
||||||
|
row["chat_id"],
|
||||||
|
row["sender_id"],
|
||||||
|
row["text"],
|
||||||
|
row["created_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PostgresRepository:
|
||||||
|
def __init__(self, *, database: Database, cache: UserCache):
|
||||||
|
self.database = database
|
||||||
|
self.cache = cache
|
||||||
|
|
||||||
|
def add_user(self, *, user: User, password_hash: str) -> User:
|
||||||
|
try:
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO users(id,username,display_name,"
|
||||||
|
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
|
||||||
|
(
|
||||||
|
user.id,
|
||||||
|
user.username,
|
||||||
|
user.display_name,
|
||||||
|
password_hash,
|
||||||
|
user.created_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except UniqueViolation:
|
||||||
|
raise DomainError(
|
||||||
|
409, "conflict", "Username already exists"
|
||||||
|
) from None
|
||||||
|
return user
|
||||||
|
|
||||||
|
def get_user(self, *, user_id: UUID) -> User | None:
|
||||||
|
cached = self.cache.get(user_id=user_id)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
|
||||||
|
(user_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
user = user_dto(row)
|
||||||
|
self.cache.put(user=user)
|
||||||
|
return user
|
||||||
|
|
||||||
|
def credentials(self, *, username: str):
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT "
|
||||||
|
+ USER_COLUMNS
|
||||||
|
+ ", password_hash FROM users WHERE username=%s",
|
||||||
|
(username,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None, None
|
||||||
|
return user_dto(row), row["password_hash"]
|
||||||
|
|
||||||
|
def add_session(
|
||||||
|
self, *, digest: bytes, user_id: UUID, created_at, expires_at
|
||||||
|
):
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sessions(token_digest,user_id,created_at,"
|
||||||
|
"expires_at) VALUES(%s,%s,%s,%s)",
|
||||||
|
(digest, user_id, created_at, expires_at),
|
||||||
|
)
|
||||||
|
|
||||||
|
def session_user(self, *, digest: bytes) -> UUID | None:
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT user_id FROM sessions WHERE token_digest=%s "
|
||||||
|
"AND expires_at>clock_timestamp()",
|
||||||
|
(digest,),
|
||||||
|
).fetchone()
|
||||||
|
return row["user_id"] if row else None
|
||||||
|
|
||||||
|
def delete_session(self, *, digest: bytes):
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
|
||||||
|
members = chat.member_ids
|
||||||
|
if actor_id not in members or len(set(members)) != len(members):
|
||||||
|
raise invalid()
|
||||||
|
try:
|
||||||
|
with self.database.connection() as conn, conn.transaction():
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
|
||||||
|
(chat.id, chat.title, chat.created_at),
|
||||||
|
)
|
||||||
|
with conn.cursor() as cursor:
|
||||||
|
cursor.executemany(
|
||||||
|
"INSERT INTO members(chat_id,user_id,position) "
|
||||||
|
"VALUES(%s,%s,%s)",
|
||||||
|
[
|
||||||
|
(chat.id, user_id, position)
|
||||||
|
for position, user_id in enumerate(members)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except ForeignKeyViolation:
|
||||||
|
raise invalid() from None
|
||||||
|
return chat
|
||||||
|
|
||||||
|
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT c.id,c.title,c.created_at,"
|
||||||
|
"array_agg(m.user_id ORDER BY m.position) member_ids "
|
||||||
|
"FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||||
|
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
|
||||||
|
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
|
||||||
|
(chat_id, actor_id),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise not_found()
|
||||||
|
return chat_dto(row)
|
||||||
|
|
||||||
|
def add_message(self, *, message: Message) -> Message:
|
||||||
|
try:
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO messages(id,chat_id,sender_id,"
|
||||||
|
"text,created_at) "
|
||||||
|
"VALUES(%s,%s,%s,%s,%s)",
|
||||||
|
(
|
||||||
|
message.id,
|
||||||
|
message.chat_id,
|
||||||
|
message.sender_id,
|
||||||
|
message.text,
|
||||||
|
message.created_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except ForeignKeyViolation:
|
||||||
|
raise not_found() from None
|
||||||
|
return message
|
||||||
|
|
||||||
|
def list_chats(
|
||||||
|
self, *, actor_id: UUID, after: Key | None, limit: int
|
||||||
|
) -> list[Chat]:
|
||||||
|
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
|
||||||
|
parameters = (
|
||||||
|
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
|
||||||
|
)
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
|
||||||
|
"FROM members WHERE chat_id=c.id "
|
||||||
|
"ORDER BY position) member_ids "
|
||||||
|
"FROM chats c JOIN members own ON own.chat_id=c.id "
|
||||||
|
"WHERE own.user_id=%s "
|
||||||
|
+ condition
|
||||||
|
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||||
|
parameters,
|
||||||
|
).fetchall()
|
||||||
|
return [chat_dto(row) for row in rows]
|
||||||
|
|
||||||
|
def list_messages(
|
||||||
|
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
|
||||||
|
) -> list[Message]:
|
||||||
|
condition = "AND (created_at,id)>(%s,%s)" if after else ""
|
||||||
|
parameters = (
|
||||||
|
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
|
||||||
|
)
|
||||||
|
with self.database.connection() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
|
||||||
|
"WHERE chat_id=%s "
|
||||||
|
+ condition
|
||||||
|
+ " ORDER BY created_at,id LIMIT %s",
|
||||||
|
parameters,
|
||||||
|
).fetchall()
|
||||||
|
return [message_dto(row) for row in rows]
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
import re
|
||||||
|
from typing import Annotated
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import (
|
||||||
|
BaseModel,
|
||||||
|
BeforeValidator,
|
||||||
|
ConfigDict,
|
||||||
|
Field,
|
||||||
|
StrictStr,
|
||||||
|
field_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
from src.cursor import parse_uuid
|
||||||
|
|
||||||
|
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
|
||||||
|
|
||||||
|
|
||||||
|
class Input(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
@field_validator("*", mode="after")
|
||||||
|
@classmethod
|
||||||
|
def nonblank(cls, value: object) -> object:
|
||||||
|
if isinstance(value, str) and (
|
||||||
|
not value.strip() or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
|
||||||
|
):
|
||||||
|
raise ValueError("Invalid text")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUser(Input):
|
||||||
|
username: StrictStr
|
||||||
|
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||||
|
password: StrictStr = Field(min_length=12, max_length=128)
|
||||||
|
|
||||||
|
@field_validator("username")
|
||||||
|
@classmethod
|
||||||
|
def username_format(cls, value: str) -> str:
|
||||||
|
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||||
|
raise ValueError("Invalid username")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class CreateChat(Input):
|
||||||
|
title: StrictStr = Field(min_length=1, max_length=100)
|
||||||
|
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMessage(Input):
|
||||||
|
text: StrictStr = Field(min_length=1, max_length=4000)
|
||||||
+157
@@ -0,0 +1,157 @@
|
|||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
from datetime import timedelta
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from src.cursor import decode_cursor, encode_cursor
|
||||||
|
from src.domain import Chat, Message, Record, User, order_key, utcnow
|
||||||
|
from src.errors import DomainError
|
||||||
|
from src.password import Passwords
|
||||||
|
from src.repository import PostgresRepository
|
||||||
|
|
||||||
|
|
||||||
|
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
|
||||||
|
items = rows[:limit]
|
||||||
|
cursor = (
|
||||||
|
encode_cursor(scope=scope, key=order_key(items[-1]))
|
||||||
|
if len(rows) > limit
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return {"items": items, "next_cursor": cursor}
|
||||||
|
|
||||||
|
|
||||||
|
def unauthorized(*, basic=False):
|
||||||
|
challenge = (
|
||||||
|
'Basic realm="messenger", charset="UTF-8"'
|
||||||
|
if basic
|
||||||
|
else 'Bearer realm="messenger"'
|
||||||
|
)
|
||||||
|
return DomainError(
|
||||||
|
401,
|
||||||
|
"unauthorized",
|
||||||
|
"Authentication required",
|
||||||
|
{"WWW-Authenticate": challenge},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def bearer(value):
|
||||||
|
if not value:
|
||||||
|
raise unauthorized()
|
||||||
|
scheme, _, token = value.partition(" ")
|
||||||
|
if scheme.lower() != "bearer" or not re.fullmatch(
|
||||||
|
r"[A-Za-z0-9_-]{32,256}", token
|
||||||
|
):
|
||||||
|
raise unauthorized()
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def token_digest(value):
|
||||||
|
return hashlib.sha256(value.encode()).digest()
|
||||||
|
|
||||||
|
|
||||||
|
class MessengerService:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
repository: PostgresRepository,
|
||||||
|
passwords: Passwords,
|
||||||
|
session_ttl: int,
|
||||||
|
) -> None:
|
||||||
|
self.repository = repository
|
||||||
|
self.passwords = passwords
|
||||||
|
self.session_ttl = session_ttl
|
||||||
|
|
||||||
|
def identify(self, *, value: str | None) -> User:
|
||||||
|
token = bearer(value)
|
||||||
|
user_id = self.repository.session_user(digest=token_digest(token))
|
||||||
|
user = self.repository.get_user(user_id=user_id) if user_id else None
|
||||||
|
if user is None:
|
||||||
|
raise unauthorized()
|
||||||
|
return user
|
||||||
|
|
||||||
|
def register(
|
||||||
|
self, *, username: str, display_name: str, password: str
|
||||||
|
) -> User:
|
||||||
|
encoded = self.passwords.hash(password=password)
|
||||||
|
return self.repository.add_user(
|
||||||
|
user=User(uuid4(), username, display_name, utcnow()),
|
||||||
|
password_hash=encoded,
|
||||||
|
)
|
||||||
|
|
||||||
|
def login(self, *, authorization: str | None):
|
||||||
|
try:
|
||||||
|
scheme, _, encoded = (authorization or "").partition(" ")
|
||||||
|
if scheme.lower() != "basic":
|
||||||
|
raise ValueError
|
||||||
|
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||||
|
username, separator, password = plain.partition(":")
|
||||||
|
if (
|
||||||
|
not separator
|
||||||
|
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
|
||||||
|
or not 12 <= len(password) <= 128
|
||||||
|
or not password.strip()
|
||||||
|
):
|
||||||
|
raise ValueError
|
||||||
|
except ValueError, UnicodeError:
|
||||||
|
raise unauthorized(basic=True) from None
|
||||||
|
user, password_hash = self.repository.credentials(username=username)
|
||||||
|
if not self.passwords.verify(password=password, encoded=password_hash):
|
||||||
|
raise unauthorized(basic=True)
|
||||||
|
now = utcnow()
|
||||||
|
expires = now + timedelta(seconds=self.session_ttl)
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
self.repository.add_session(
|
||||||
|
digest=token_digest(token),
|
||||||
|
user_id=user.id,
|
||||||
|
created_at=now,
|
||||||
|
expires_at=expires,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"token": token,
|
||||||
|
"token_type": "Bearer",
|
||||||
|
"expires_at": expires,
|
||||||
|
"user": user,
|
||||||
|
}
|
||||||
|
|
||||||
|
def logout(self, *, authorization: str | None):
|
||||||
|
self.identify(value=authorization)
|
||||||
|
self.repository.delete_session(
|
||||||
|
digest=token_digest(bearer(authorization))
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_chat(
|
||||||
|
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
|
||||||
|
) -> Chat:
|
||||||
|
return self.repository.add_chat(
|
||||||
|
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
|
||||||
|
actor_id=actor_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
|
||||||
|
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
|
||||||
|
|
||||||
|
def send(self, *, actor_id: UUID, chat_id: UUID, text: str) -> Message:
|
||||||
|
return self.repository.add_message(
|
||||||
|
message=Message(uuid4(), chat_id, actor_id, text, utcnow())
|
||||||
|
)
|
||||||
|
|
||||||
|
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||||
|
scope = f"chats:{actor_id}"
|
||||||
|
after = decode_cursor(value=cursor, scope=scope)
|
||||||
|
rows = self.repository.list_chats(
|
||||||
|
actor_id=actor_id, after=after, limit=limit
|
||||||
|
)
|
||||||
|
return page(rows=rows, limit=limit, scope=scope)
|
||||||
|
|
||||||
|
def history(
|
||||||
|
self, *, actor_id: UUID, chat_id: UUID, limit: int, cursor: str | None
|
||||||
|
) -> dict:
|
||||||
|
self.get_chat(actor_id=actor_id, chat_id=chat_id)
|
||||||
|
scope = f"messages:{chat_id}"
|
||||||
|
after = decode_cursor(value=cursor, scope=scope)
|
||||||
|
rows = self.repository.list_messages(
|
||||||
|
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
|
||||||
|
)
|
||||||
|
return page(rows=rows, limit=limit, scope=scope)
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import os
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def secret(name: str) -> str:
|
||||||
|
file = os.environ.get(name + "_FILE")
|
||||||
|
if file:
|
||||||
|
return Path(file).read_text().strip()
|
||||||
|
value = os.environ.get(name)
|
||||||
|
if not value:
|
||||||
|
raise ValueError("Missing required secret setting: " + name)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def integer(name: str, default: int, low: int, high: int) -> int:
|
||||||
|
value = int(os.environ.get(name, str(default)))
|
||||||
|
if not low <= value <= high:
|
||||||
|
raise ValueError("Invalid setting: " + name)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Settings:
|
||||||
|
database_url: str = field(repr=False)
|
||||||
|
cache_url: str = field(repr=False)
|
||||||
|
session_ttl: int = 3600
|
||||||
|
pool_size: int = 10
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(cls):
|
||||||
|
return cls(
|
||||||
|
secret("DATABASE_URL"),
|
||||||
|
secret("CACHE_URL"),
|
||||||
|
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
|
||||||
|
integer("DB_POOL_SIZE", 10, 1, 32),
|
||||||
|
)
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from jsonschema import Draft202012Validator, FormatChecker
|
||||||
|
|
||||||
|
CONTRACT = json.loads(
|
||||||
|
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
|
||||||
|
)
|
||||||
|
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8082")
|
||||||
|
PASSWORD = "Synthetic-Password-🌍-123"
|
||||||
|
TOKENS = {}
|
||||||
|
|
||||||
|
|
||||||
|
def check(result: httpx.Response, status: int, schema: str) -> dict:
|
||||||
|
assert result.status_code == status, result.text
|
||||||
|
assert result.headers["x-request-id"]
|
||||||
|
assert result.headers["content-type"].startswith("application/json")
|
||||||
|
payload = result.json()
|
||||||
|
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
|
||||||
|
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
|
||||||
|
payload
|
||||||
|
)
|
||||||
|
if schema == "Error":
|
||||||
|
assert payload["error"]["request_id"] == result.headers["x-request-id"]
|
||||||
|
assert (
|
||||||
|
payload["error"]["code"]
|
||||||
|
== {
|
||||||
|
400: "invalid_request",
|
||||||
|
401: "unauthorized",
|
||||||
|
404: "not_found",
|
||||||
|
409: "conflict",
|
||||||
|
500: "internal_error",
|
||||||
|
}[status]
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def register(client: httpx.Client, **overrides) -> dict:
|
||||||
|
data = {
|
||||||
|
"username": "u_" + uuid4().hex[:24],
|
||||||
|
"display_name": "Алиса 🌍",
|
||||||
|
"password": PASSWORD,
|
||||||
|
}
|
||||||
|
data.update(overrides)
|
||||||
|
user = check(client.post("/api/v1/users", json=data), 201, "User")
|
||||||
|
session = check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
auth=httpx.BasicAuth(data["username"], data["password"]),
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Session",
|
||||||
|
)
|
||||||
|
TOKENS[user["id"]] = session["token"]
|
||||||
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client():
|
||||||
|
with httpx.Client(
|
||||||
|
base_url=BASE_URL, timeout=15, trust_env=False
|
||||||
|
) as client:
|
||||||
|
yield client
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def actors(client):
|
||||||
|
return register(client), register(client), register(client)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def chat(client, actors):
|
||||||
|
alice, bob, _ = actors
|
||||||
|
return check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers(alice),
|
||||||
|
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Chat",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def headers(actor: dict) -> dict:
|
||||||
|
return {"Authorization": "Bearer " + TOKENS[actor["id"]]}
|
||||||
|
|
||||||
|
|
||||||
|
def history(client, actor, chat_id, limit=7):
|
||||||
|
found, cursor = [], None
|
||||||
|
while True:
|
||||||
|
params = {"limit": limit}
|
||||||
|
if cursor:
|
||||||
|
params["cursor"] = cursor
|
||||||
|
page = check(
|
||||||
|
client.get(
|
||||||
|
f"/api/v1/chats/{chat_id}/messages",
|
||||||
|
headers=headers(actor),
|
||||||
|
params=params,
|
||||||
|
),
|
||||||
|
200,
|
||||||
|
"MessagePage",
|
||||||
|
)
|
||||||
|
found.extend(page["items"])
|
||||||
|
cursor = page["next_cursor"]
|
||||||
|
if cursor is None:
|
||||||
|
return found
|
||||||
@@ -0,0 +1,412 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from conftest import BASE_URL, PASSWORD, check, headers, history, register
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
|
||||||
|
def test_health(client, path):
|
||||||
|
assert check(client.get(path), 200, "Health") == {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
|
||||||
|
)
|
||||||
|
def test_identification(client, value):
|
||||||
|
check(
|
||||||
|
client.get(
|
||||||
|
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
|
||||||
|
),
|
||||||
|
401,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_identity(client, actors):
|
||||||
|
assert (
|
||||||
|
check(
|
||||||
|
client.get("/api/v1/users/me", headers=headers(actors[0])),
|
||||||
|
200,
|
||||||
|
"User",
|
||||||
|
)
|
||||||
|
== actors[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"data",
|
||||||
|
[
|
||||||
|
{},
|
||||||
|
[],
|
||||||
|
None,
|
||||||
|
{"username": "ABC", "display_name": "A"},
|
||||||
|
{"username": "abc\n", "display_name": "A"},
|
||||||
|
{"username": 123, "display_name": "A"},
|
||||||
|
{"username": "abc", "display_name": " "},
|
||||||
|
{"username": "abc", "display_name": "x" * 101},
|
||||||
|
{"username": "abc", "display_name": "A", "extra": True},
|
||||||
|
{"username": "abc", "display_name": "\ud800"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_invalid_user(client, data):
|
||||||
|
if isinstance(data, dict):
|
||||||
|
data = {"password": PASSWORD, **data}
|
||||||
|
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
|
||||||
|
)
|
||||||
|
def test_malformed_json(client, raw):
|
||||||
|
check(client.post("/api/v1/users", content=raw), 400, "Error")
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth_before_json(client):
|
||||||
|
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
|
||||||
|
|
||||||
|
|
||||||
|
def test_duplicate_username(client, actors):
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/users",
|
||||||
|
json={
|
||||||
|
"username": actors[0]["username"],
|
||||||
|
"display_name": "B",
|
||||||
|
"password": PASSWORD,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
409,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unicode_codepoints(client, actors, chat):
|
||||||
|
user = register(client, display_name="🌍" * 100)
|
||||||
|
assert user["display_name"] == "🌍" * 100
|
||||||
|
text = "🌍" * 4000
|
||||||
|
result = check(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/chats/{chat['id']}/messages",
|
||||||
|
headers=headers(actors[0]),
|
||||||
|
json={"text": text},
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Message",
|
||||||
|
)
|
||||||
|
assert result["text"] == text
|
||||||
|
assert result["sender_id"] == actors[0]["id"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"method,suffix,body",
|
||||||
|
[
|
||||||
|
("GET", "", None),
|
||||||
|
("GET", "/messages", None),
|
||||||
|
("POST", "/messages", {"text": "hello"}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_acl(client, actors, chat, method, suffix, body):
|
||||||
|
outsider = headers(actors[2])
|
||||||
|
actual = check(
|
||||||
|
client.request(
|
||||||
|
method,
|
||||||
|
f"/api/v1/chats/{chat['id']}{suffix}",
|
||||||
|
headers=outsider,
|
||||||
|
json=body,
|
||||||
|
),
|
||||||
|
404,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
missing = check(
|
||||||
|
client.request(
|
||||||
|
method,
|
||||||
|
f"/api/v1/chats/{uuid4()}{suffix}",
|
||||||
|
headers=outsider,
|
||||||
|
json=body,
|
||||||
|
),
|
||||||
|
404,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
assert actual["error"]["message"] == missing["error"]["message"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
|
||||||
|
def test_bad_message(client, actors, chat, value):
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/chats/{chat['id']}/messages",
|
||||||
|
headers=headers(actors[0]),
|
||||||
|
json={"text": value},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sender_not_client_controlled(client, actors, chat):
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/chats/{chat['id']}/messages",
|
||||||
|
headers=headers(actors[0]),
|
||||||
|
json={"text": "Hi", "sender_id": actors[1]["id"]},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"case",
|
||||||
|
[
|
||||||
|
"duplicate",
|
||||||
|
"unknown",
|
||||||
|
"no_caller",
|
||||||
|
"empty",
|
||||||
|
"wrong_type",
|
||||||
|
"bad_uuid",
|
||||||
|
"too_many",
|
||||||
|
"blank_title",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_invalid_chat(client, actors, case):
|
||||||
|
a, b, _ = actors
|
||||||
|
members = {
|
||||||
|
"duplicate": [a["id"], a["id"]],
|
||||||
|
"unknown": [a["id"], str(uuid4())],
|
||||||
|
"no_caller": [b["id"]],
|
||||||
|
"empty": [],
|
||||||
|
"wrong_type": a["id"],
|
||||||
|
"bad_uuid": ["broken"],
|
||||||
|
"too_many": [str(uuid4()) for _ in range(101)],
|
||||||
|
"blank_title": [a["id"]],
|
||||||
|
}[case]
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers(a),
|
||||||
|
json={
|
||||||
|
"title": " " if case == "blank_title" else "Title",
|
||||||
|
"member_ids": members,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
|
||||||
|
)
|
||||||
|
def test_bad_limit(client, actors, chat, limit):
|
||||||
|
check(
|
||||||
|
client.get(
|
||||||
|
f"/api/v1/chats/{chat['id']}/messages",
|
||||||
|
headers=headers(actors[0]),
|
||||||
|
params={"limit": limit},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
|
||||||
|
)
|
||||||
|
def test_bad_cursor(client, actors, chat, cursor):
|
||||||
|
check(
|
||||||
|
client.get(
|
||||||
|
f"/api/v1/chats/{chat['id']}/messages",
|
||||||
|
headers=headers(actors[0]),
|
||||||
|
params={"cursor": cursor},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pagination(client, actors, chat):
|
||||||
|
a, b, outsider = actors
|
||||||
|
sent = []
|
||||||
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||||
|
for i in range(23):
|
||||||
|
sent.append(
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
path,
|
||||||
|
headers=headers(a if i % 2 else b),
|
||||||
|
json={"text": f"Msg {i}"},
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Message",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
got = history(client, a, chat["id"])
|
||||||
|
assert got == sorted(
|
||||||
|
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
|
||||||
|
)
|
||||||
|
first = check(
|
||||||
|
client.get(path, headers=headers(a), params={"limit": 1}),
|
||||||
|
200,
|
||||||
|
"MessagePage",
|
||||||
|
)
|
||||||
|
second = check(
|
||||||
|
client.get(
|
||||||
|
path,
|
||||||
|
headers=headers(b),
|
||||||
|
params={"limit": 22, "cursor": first["next_cursor"]},
|
||||||
|
),
|
||||||
|
200,
|
||||||
|
"MessagePage",
|
||||||
|
)
|
||||||
|
assert second["items"] == got[1:]
|
||||||
|
assert second["next_cursor"] is None
|
||||||
|
chats = check(
|
||||||
|
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
|
||||||
|
)
|
||||||
|
assert chats["items"] == []
|
||||||
|
other = check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers(a),
|
||||||
|
json={"title": "Other", "member_ids": [a["id"]]},
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Chat",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
client.get(
|
||||||
|
f"/api/v1/chats/{other['id']}/messages",
|
||||||
|
headers=headers(a),
|
||||||
|
params={"cursor": first["next_cursor"]},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
|
||||||
|
data = json.loads(
|
||||||
|
base64.urlsafe_b64decode(
|
||||||
|
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
data["at"] = 17
|
||||||
|
forged = (
|
||||||
|
base64.urlsafe_b64encode(json.dumps(data).encode())
|
||||||
|
.decode()
|
||||||
|
.rstrip("=")
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
client.get(path, headers=headers(a), params={"cursor": forged}),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_pagination_scope(client, actors):
|
||||||
|
a, b, _ = actors
|
||||||
|
created = []
|
||||||
|
for i in range(4):
|
||||||
|
created.append(
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers(a),
|
||||||
|
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Chat",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
page = check(
|
||||||
|
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
|
||||||
|
200,
|
||||||
|
"ChatPage",
|
||||||
|
)
|
||||||
|
assert page["items"] == created[:2]
|
||||||
|
last = check(
|
||||||
|
client.get(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers(a),
|
||||||
|
params={"limit": 2, "cursor": page["next_cursor"]},
|
||||||
|
),
|
||||||
|
200,
|
||||||
|
"ChatPage",
|
||||||
|
)
|
||||||
|
assert last["items"] == created[2:]
|
||||||
|
assert last["next_cursor"] is None
|
||||||
|
check(
|
||||||
|
client.get(
|
||||||
|
"/api/v1/chats",
|
||||||
|
headers=headers(b),
|
||||||
|
params={"cursor": page["next_cursor"]},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concurrent_messages(client, actors, chat):
|
||||||
|
barrier = threading.Barrier(10)
|
||||||
|
a = actors[0]
|
||||||
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||||
|
|
||||||
|
def worker(i):
|
||||||
|
with httpx.Client(
|
||||||
|
base_url=BASE_URL, timeout=15, trust_env=False
|
||||||
|
) as conn:
|
||||||
|
barrier.wait(timeout=15)
|
||||||
|
return [
|
||||||
|
check(
|
||||||
|
conn.post(
|
||||||
|
path, headers=headers(a), json={"text": f"{i}:{j}"}
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Message",
|
||||||
|
)["id"]
|
||||||
|
for j in range(10)
|
||||||
|
]
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||||
|
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
|
||||||
|
rows = history(client, a, chat["id"])
|
||||||
|
assert len(rows) == len(ids) == len(set(ids)) == 100
|
||||||
|
assert set(ids) == {row["id"] for row in rows}
|
||||||
|
assert len({row["text"] for row in rows}) == 100
|
||||||
|
|
||||||
|
|
||||||
|
def test_registration_race(client):
|
||||||
|
username = "race_" + uuid4().hex[:20]
|
||||||
|
barrier = threading.Barrier(10)
|
||||||
|
|
||||||
|
def worker(_):
|
||||||
|
with httpx.Client(
|
||||||
|
base_url=BASE_URL, timeout=15, trust_env=False
|
||||||
|
) as conn:
|
||||||
|
barrier.wait(timeout=15)
|
||||||
|
result = conn.post(
|
||||||
|
"/api/v1/users",
|
||||||
|
json={
|
||||||
|
"username": username,
|
||||||
|
"display_name": "Test",
|
||||||
|
"password": PASSWORD,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
result,
|
||||||
|
result.status_code,
|
||||||
|
"User" if result.status_code == 201 else "Error",
|
||||||
|
)
|
||||||
|
return result.status_code
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||||
|
statuses = list(pool.map(worker, range(10)))
|
||||||
|
assert statuses.count(201) == 1
|
||||||
|
assert statuses.count(409) == 9
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_path(client):
|
||||||
|
check(client.get("/does-not-exist"), 404, "Error")
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import json
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from conftest import check, headers
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
|
||||||
|
def test_only_utf8_json(client, encoding):
|
||||||
|
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
|
||||||
|
payload["password"] = "Synthetic-encoding-test-123"
|
||||||
|
result = client.post(
|
||||||
|
"/api/v1/users",
|
||||||
|
content=json.dumps(payload).encode(encoding),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
check(result, 400, "Error")
|
||||||
|
|
||||||
|
|
||||||
|
def test_leading_zero_limit(client, actors, chat):
|
||||||
|
result = client.get(
|
||||||
|
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||||
|
headers=headers(actors[0]),
|
||||||
|
params={"limit": "0001"},
|
||||||
|
)
|
||||||
|
check(result, 200, "MessagePage")
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from conftest import PASSWORD, TOKENS, check, headers, register
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
|
||||||
|
)
|
||||||
|
def test_password_validation(client, password):
|
||||||
|
check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/users",
|
||||||
|
json={
|
||||||
|
"username": "pwd_" + uuid4().hex[:20],
|
||||||
|
"display_name": "Test",
|
||||||
|
"password": password,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
400,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_utf8_password(client):
|
||||||
|
password = "🌍" * 127 + ":"
|
||||||
|
user = register(client, password=password)
|
||||||
|
assert (
|
||||||
|
check(
|
||||||
|
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
|
||||||
|
)
|
||||||
|
== user
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
|
||||||
|
)
|
||||||
|
def test_bad_basic(client, authorization):
|
||||||
|
result = client.post(
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
headers={"Authorization": authorization} if authorization else {},
|
||||||
|
)
|
||||||
|
check(result, 401, "Error")
|
||||||
|
assert (
|
||||||
|
result.headers["www-authenticate"]
|
||||||
|
== 'Basic realm="messenger", charset="UTF-8"'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_wrong_password_and_unknown_user(client):
|
||||||
|
user = register(client)
|
||||||
|
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
|
||||||
|
result = client.post(
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
|
||||||
|
)
|
||||||
|
data = check(result, 401, "Error")
|
||||||
|
assert data["error"]["message"] == "Authentication required"
|
||||||
|
|
||||||
|
|
||||||
|
def test_logout_is_not_revived_by_cache(client):
|
||||||
|
user = register(client)
|
||||||
|
auth = headers(user)
|
||||||
|
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
|
||||||
|
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
|
||||||
|
assert result.status_code == 204 and result.content == b""
|
||||||
|
assert result.headers["x-request-id"]
|
||||||
|
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
|
||||||
|
check(
|
||||||
|
client.delete("/api/v1/auth/sessions/current", headers=auth),
|
||||||
|
401,
|
||||||
|
"Error",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_current_session_revoked(client):
|
||||||
|
user = register(client)
|
||||||
|
another = check(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
auth=httpx.BasicAuth(user["username"], PASSWORD),
|
||||||
|
),
|
||||||
|
201,
|
||||||
|
"Session",
|
||||||
|
)
|
||||||
|
assert another["token"] != TOKENS[user["id"]]
|
||||||
|
assert len(another["token"]) >= 32
|
||||||
|
assert (
|
||||||
|
client.delete(
|
||||||
|
"/api/v1/auth/sessions/current", headers=headers(user)
|
||||||
|
).status_code
|
||||||
|
== 204
|
||||||
|
)
|
||||||
|
result = client.get(
|
||||||
|
"/api/v1/users/me",
|
||||||
|
headers={"Authorization": "Bearer " + another["token"]},
|
||||||
|
)
|
||||||
|
assert check(result, 200, "User") == user
|
||||||
|
|
||||||
|
|
||||||
|
def test_bearer_source_and_ignored_user_header(client, actors):
|
||||||
|
alice, bob, _ = actors
|
||||||
|
auth = {**headers(alice), "X-User-Id": bob["id"]}
|
||||||
|
assert (
|
||||||
|
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
|
||||||
|
== alice["id"]
|
||||||
|
)
|
||||||
|
result = client.get(
|
||||||
|
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
|
||||||
|
)
|
||||||
|
check(result, 401, "Error")
|
||||||
|
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'
|
||||||
Reference in New Issue
Block a user