forked from mtusi2026/mtusi2026lab2
Complete lab
This commit is contained in:
+8
-8
@@ -1,12 +1,12 @@
|
||||
# Used by python -m src. Docker run reads this file through --env-file.
|
||||
# Copy to .env; choose a local password. Never commit .env.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
# Single-process resource limits. Text bytes are UTF-8 payload, not process RSS.
|
||||
MAX_USERS=1000
|
||||
MAX_CHATS=1000
|
||||
MAX_MESSAGES=10000
|
||||
MAX_TEXT_BYTES=16777216
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace-with-local-password
|
||||
SESSION_TTL_SECONDS=3600
|
||||
CACHE_TTL_SECONDS=30
|
||||
MAX_REQUEST_BYTES=65536
|
||||
MAX_INFLIGHT=32
|
||||
MESSAGE_RATE=1000
|
||||
MESSAGE_BURST=1000
|
||||
# Compose sets POSTGRES_HOST=postgres and REDIS_URL=redis://redis:6379/0.
|
||||
# For a manual API launch set DATABASE_URL and REDIS_URL in the environment.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
*.sh text eol=lf
|
||||
@@ -1,18 +1,21 @@
|
||||
## Лабораторная 2
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: 1 — согласованность кэша
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: FastAPI и REST API из lab1; PostgreSQL,
|
||||
SQL-миграции, Argon2id, Basic/Bearer, Redis-сессии с TTL и AOF, Compose.
|
||||
Трек 1: кэш истории с версией из PostgreSQL, воспроизведение stale read,
|
||||
защита и измерения SQL/latency; read-after-write и восстановление после рестарта Redis.
|
||||
Команды запуска из чистого clone: [REPORT.md](../REPORT.md).
|
||||
Результаты проверок: [REPORT.md](../REPORT.md), [evidence](../evidence).
|
||||
Ограничения: один Docker-host, без TLS и резервных копий; только локальный стенд.
|
||||
Источники и использование ИИ: документация библиотек и помощь Codex, подробнее в отчёте.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] Эквиваленты make check и make test выполнены через Python на Windows; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: Lab 2 application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
integration:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Prepare local configuration
|
||||
run: |
|
||||
python -c "from pathlib import Path; import secrets; Path('.env').write_text(Path('.env.example').read_text().replace('replace-with-local-password', secrets.token_hex(24)))"
|
||||
chmod +x scripts/restart.sh
|
||||
- run: docker compose up -d --build --wait --wait-timeout 120
|
||||
- run: make check test test-grade4 PYTHON=python
|
||||
- run: make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.sh
|
||||
- run: make test-failures explain PYTHON=python
|
||||
- run: make test-track1 PYTHON=python
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
@@ -12,6 +12,11 @@ RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY contracts ./contracts
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
COPY tests ./tests
|
||||
COPY student-tests ./student-tests
|
||||
COPY scripts ./scripts
|
||||
COPY lab.json .
|
||||
|
||||
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
||||
USER appuser
|
||||
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence
|
||||
.PHONY: help check validate test load test-persistence test-grade4 test-track1 test-failures explain
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -24,3 +24,15 @@ load:
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-grade4:
|
||||
docker compose exec -T api python -m unittest discover -s student-tests -v
|
||||
|
||||
test-failures:
|
||||
$(PYTHON) student-tests/failures.py
|
||||
|
||||
explain:
|
||||
docker compose exec -T api python -m scripts.explain_history
|
||||
|
||||
test-track1:
|
||||
$(PYTHON) scripts/track1.py
|
||||
|
||||
@@ -1,37 +1,82 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Ромашко Владимир, БВТ2301
|
||||
- Целевая оценка: 5, трек 1 — согласованность кэша
|
||||
- Commit/PR: указать после публикации
|
||||
- Источники и помощь ИИ: документация PostgreSQL, Redis, SQLAlchemy; использован ChatGPT
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуется Docker Desktop/Engine с Compose v2 и Python 3.9+.
|
||||
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
# Задать POSTGRES_PASSWORD в .env
|
||||
docker compose up -d --build --wait --wait-timeout 120
|
||||
python scripts/check_contract.py
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
docker compose exec -T api python -m unittest discover -s student-tests -v
|
||||
python tests/resilience.py --mode persistence --base-url http://127.0.0.1:8080 --action-script scripts/restart.cmd
|
||||
python student-tests/failures.py
|
||||
python scripts/track1.py
|
||||
```
|
||||
|
||||
Для Linux перед запуском: `chmod +x scripts/restart.sh`.
|
||||
|
||||
Остановка без удаления данных: `docker compose down`.
|
||||
Удаление учебных volumes: `docker compose down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент → FastAPI → PostgreSQL и Redis.
|
||||
|
||||
PostgreSQL — источник истины для пользователей, чатов, участников и сообщений.
|
||||
SQLAlchemy ORM использует пул до 10 соединений, timeout пула 2 с,
|
||||
connect timeout 2 с, statement timeout 3 с.
|
||||
|
||||
Redis хранит сессии с TTL и кэш страниц истории. Включены AOF,
|
||||
`appendfsync always` и именованный volume.
|
||||
|
||||
Пароли хешируются Argon2id. Basic используется только для создания сессии,
|
||||
остальные защищённые запросы используют Bearer. `X-User-Id` игнорируется.
|
||||
|
||||
Миграции находятся в `migrations/`. Создание чата и участников выполняется
|
||||
в одной транзакции. При создании сообщения версия истории увеличивается
|
||||
в той же транзакции.
|
||||
|
||||
Ключ кэша истории содержит `chat_id`, версию истории, limit и cursor.
|
||||
После записи старый ключ не используется. При ошибке кэша чтение выполняется
|
||||
из PostgreSQL.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `python tests/smoke.py` | PASS | `evidence/track1-regression.txt` |
|
||||
| Сохранность после рестарта | `tests/resilience.py` | Сессия и сообщения сохранены | `evidence/track1-regression.txt` |
|
||||
| Миграции, ORM и транзакции | `student-tests` | 13 тестов OK; проверены rollback, гонка регистрации и повторные миграции | `evidence/orm-tests.txt` |
|
||||
| Redis, TTL и отказы | `student-tests/failures.py` | Redis/PostgreSQL возвращают 503 и восстанавливаются | `evidence/track1-failures.txt` |
|
||||
| Индекс истории | `python -m scripts.explain_history` | Index Scan по `messages_history_idx` на 10 000 сообщений | `evidence/explain.txt` |
|
||||
| Трек 1: stale read | `python scripts/track1.py` | До защиты 5/5 stale read, после защиты 0/5 | `evidence/cache-experiment.json` |
|
||||
| Трек 1: производительность | `python scripts/track1.py` | SQL-команды: 600 без кэша и 300 с кэшем | `evidence/cache-experiment.json` |
|
||||
| Трек 1: восстановление | `python scripts/track1.py` | Redis перезапущен, read-after-write и logout сохранены | `evidence/cache-restart.json` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза: версия истории в ключе Redis устранит stale read при позднем заполнении кэша.
|
||||
|
||||
Методика: 5 повторов управляемой конкурентной гонки; затем 3 серии по 300 чтений,
|
||||
concurrency 8, страница 100 сообщений, отдельный чат с 10 000 сообщениями.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Конкурентная запись и чтение | Старый ключ не должен использоваться | 0/5 stale read после защиты | Не применимо |
|
||||
| Redis stop/restart | Ошибка 503, затем восстановление | 503; сессия и сообщения доступны | 6.1 с, потери нет |
|
||||
| PostgreSQL stop/start | Ошибка 503, затем восстановление | 503; API не доверяет анонимному запросу | 6.2 с, потери нет |
|
||||
| Redis restart в треке 1 | Кэш восстановится из PostgreSQL | Read-after-write и logout прошли | 1.2 с, потери нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Стенд рассчитан на один Docker-host и один экземпляр PostgreSQL/Redis.
|
||||
Cache stampede не исследовался. Нагрузочные результаты относятся к слою хранения,
|
||||
а не ко всему HTTP API.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
services:
|
||||
api:
|
||||
build: .
|
||||
dns_opt:
|
||||
- timeout:1
|
||||
- attempts:1
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
POSTGRES_HOST: postgres
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
|
||||
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
|
||||
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
|
||||
postgres:
|
||||
image: postgres:17.4-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-bookworm
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,207 @@
|
||||
{
|
||||
"race_repeats": 5,
|
||||
"race": {
|
||||
"naive": [
|
||||
{
|
||||
"stale": true,
|
||||
"read_after_write_ms": 3.362,
|
||||
"sql": 6,
|
||||
"history_selects": 1
|
||||
},
|
||||
{
|
||||
"stale": true,
|
||||
"read_after_write_ms": 2.705,
|
||||
"sql": 6,
|
||||
"history_selects": 1
|
||||
},
|
||||
{
|
||||
"stale": true,
|
||||
"read_after_write_ms": 2.644,
|
||||
"sql": 6,
|
||||
"history_selects": 1
|
||||
},
|
||||
{
|
||||
"stale": true,
|
||||
"read_after_write_ms": 2.601,
|
||||
"sql": 6,
|
||||
"history_selects": 1
|
||||
},
|
||||
{
|
||||
"stale": true,
|
||||
"read_after_write_ms": 2.491,
|
||||
"sql": 6,
|
||||
"history_selects": 1
|
||||
}
|
||||
],
|
||||
"versioned": [
|
||||
{
|
||||
"stale": false,
|
||||
"read_after_write_ms": 6.639,
|
||||
"sql": 7,
|
||||
"history_selects": 2
|
||||
},
|
||||
{
|
||||
"stale": false,
|
||||
"read_after_write_ms": 7.862,
|
||||
"sql": 7,
|
||||
"history_selects": 2
|
||||
},
|
||||
{
|
||||
"stale": false,
|
||||
"read_after_write_ms": 8.104,
|
||||
"sql": 7,
|
||||
"history_selects": 2
|
||||
},
|
||||
{
|
||||
"stale": false,
|
||||
"read_after_write_ms": 8.303,
|
||||
"sql": 7,
|
||||
"history_selects": 2
|
||||
},
|
||||
{
|
||||
"stale": false,
|
||||
"read_after_write_ms": 6.384,
|
||||
"sql": 7,
|
||||
"history_selects": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
"benchmark": {
|
||||
"uncached": [
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 2.84,
|
||||
"successful_rps": 105.64,
|
||||
"client_cpu_seconds": 2.878,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 600,
|
||||
"history_selects": 300,
|
||||
"p50_ms": 78.194,
|
||||
"p95_ms": 108.779,
|
||||
"p99_ms": 140.556
|
||||
},
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 3.164,
|
||||
"successful_rps": 94.82,
|
||||
"client_cpu_seconds": 3.148,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 600,
|
||||
"history_selects": 300,
|
||||
"p50_ms": 85.254,
|
||||
"p95_ms": 126.226,
|
||||
"p99_ms": 150.227
|
||||
},
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 3.134,
|
||||
"successful_rps": 95.73,
|
||||
"client_cpu_seconds": 3.142,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 600,
|
||||
"history_selects": 300,
|
||||
"p50_ms": 83.202,
|
||||
"p95_ms": 121.385,
|
||||
"p99_ms": 148.622
|
||||
}
|
||||
],
|
||||
"naive": [
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.683,
|
||||
"successful_rps": 178.21,
|
||||
"client_cpu_seconds": 1.689,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 300,
|
||||
"history_selects": 0,
|
||||
"p50_ms": 39.913,
|
||||
"p95_ms": 86.212,
|
||||
"p99_ms": 101.446
|
||||
},
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.361,
|
||||
"successful_rps": 220.37,
|
||||
"client_cpu_seconds": 1.353,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 300,
|
||||
"history_selects": 0,
|
||||
"p50_ms": 28.237,
|
||||
"p95_ms": 65.387,
|
||||
"p99_ms": 73.071
|
||||
},
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.147,
|
||||
"successful_rps": 261.59,
|
||||
"client_cpu_seconds": 1.17,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 300,
|
||||
"history_selects": 0,
|
||||
"p50_ms": 22.607,
|
||||
"p95_ms": 58.264,
|
||||
"p99_ms": 63.041
|
||||
}
|
||||
],
|
||||
"versioned": [
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.238,
|
||||
"successful_rps": 242.35,
|
||||
"client_cpu_seconds": 1.254,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 300,
|
||||
"history_selects": 0,
|
||||
"p50_ms": 25.229,
|
||||
"p95_ms": 60.546,
|
||||
"p99_ms": 65.823
|
||||
},
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.696,
|
||||
"successful_rps": 176.85,
|
||||
"client_cpu_seconds": 1.709,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 300,
|
||||
"history_selects": 0,
|
||||
"p50_ms": 41.596,
|
||||
"p95_ms": 73.311,
|
||||
"p99_ms": 81.611
|
||||
},
|
||||
{
|
||||
"requests": 300,
|
||||
"concurrency": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.209,
|
||||
"successful_rps": 248.18,
|
||||
"client_cpu_seconds": 1.22,
|
||||
"client_peak_rss_mib": 132.52,
|
||||
"sql": 300,
|
||||
"history_selects": 0,
|
||||
"p50_ms": 24.794,
|
||||
"p95_ms": 58.879,
|
||||
"p99_ms": 64.291
|
||||
}
|
||||
]
|
||||
},
|
||||
"messages": 10000,
|
||||
"page_size": 100,
|
||||
"warmup_reads_per_run": 1,
|
||||
"scope": "store calls; real SQL and Redis; excludes HTTP, sessions and KDF; SQL excludes BEGIN/COMMIT"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
Post-experiment snapshot (not peak):
|
||||
mtusi2026lab2-api-1 CPU=0.22% RAM=58.04MiB / 512MiB
|
||||
mtusi2026lab2-postgres-1 CPU=7.08% RAM=38.14MiB / 512MiB
|
||||
mtusi2026lab2-redis-1 CPU=0.30% RAM=3.582MiB / 256MiB
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"same_session": true,
|
||||
"confirmed_messages": 3,
|
||||
"cold_cache_rebuilt": true,
|
||||
"read_after_write": true,
|
||||
"logout_still_401": true,
|
||||
"recovery_seconds": 1.161,
|
||||
"redis_start_before_utc": "2026-10-09T12:36:26.91215174Z",
|
||||
"redis_start_after_utc": "2026-10-09T12:47:22.636577776Z"
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
Date: 2026-10-09 (Europe/Moscow)
|
||||
|
||||
Command: python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
Command: docker compose exec -T api python tests/smoke.py
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
Command: docker compose exec -T api python -m unittest discover -s student-tests -v
|
||||
test_chat_rollback ... ok
|
||||
test_expiry_short_ttl ... ok
|
||||
test_migrations_repeat ... ok
|
||||
test_password_logout_and_identity ... ok
|
||||
test_registration_race ... ok
|
||||
test_unicode_password ... ok
|
||||
Ran 6 tests in 8.057s
|
||||
OK
|
||||
|
||||
Command: python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
Container start times: evidence/restart.txt
|
||||
|
||||
Command: python student-tests/failures.py
|
||||
PASS redis unavailable: 503; seconds=[3.04, 3.041, 3.036]; anonymous=401; live=200
|
||||
PASS redis recovered in 6.12s: same session and message
|
||||
PASS postgres unavailable: 503; seconds=[2.032, 4.034, 4.049]; anonymous=401; live=200
|
||||
PASS postgres recovered in 6.41s: same session and message
|
||||
|
||||
Initial failure test exceeded the HTTP timeout because Docker DNS waited too long
|
||||
for the stopped service. Fixed with dns_opt timeout:1, attempts:1 on the API.
|
||||
The timings above are from the successful rerun after this fix.
|
||||
|
||||
Final verification after fixes:
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
Integration suite repeated: Ran 6 tests in 8.030s; OK.
|
||||
docker compose ps: api, postgres and redis are healthy.
|
||||
@@ -0,0 +1,12 @@
|
||||
Command: docker compose exec -T api python -m scripts.explain_history
|
||||
Dataset: 10000 messages; page after message 9000; ORM query; limit 51
|
||||
Limit (cost=0.29..75.12 rows=51 width=1055) (actual time=0.013..0.028 rows=51 loops=1)
|
||||
Buffers: shared hit=10
|
||||
-> Index Scan using messages_history_idx on messages (cost=0.29..1535.22 rows=1046 width=1055) (actual time=0.013..0.024 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = 'ff693a1b-4e99-43c9-bb51-68b5acdd972b'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:09+00'::timestamp with time zone, 'e35d4a84-afe1-4aae-8f55-27be1490c84e'::uuid)))
|
||||
Buffers: shared hit=10
|
||||
Planning:
|
||||
Buffers: shared hit=7
|
||||
Planning Time: 0.078 ms
|
||||
Execution Time: 0.056 ms
|
||||
Synthetic data rolled back.
|
||||
@@ -0,0 +1,20 @@
|
||||
SQLAlchemy ORM
|
||||
Command: docker compose exec -T api python -m unittest discover -s student-tests -v
|
||||
test_concurrent_writers_do_not_lose_version_increments (test_cache.CacheTests.test_concurrent_writers_do_not_lose_version_increments) ... ok
|
||||
test_hit_still_checks_membership_and_cache_failure_falls_back (test_cache.CacheTests.test_hit_still_checks_membership_and_cache_failure_falls_back) ... ok
|
||||
test_http_read_after_write_and_cursor_pages (test_cache.CacheTests.test_http_read_after_write_and_cursor_pages) ... ok
|
||||
test_late_fill_cannot_restore_old_page (test_cache.CacheTests.test_late_fill_cannot_restore_old_page) ... ok
|
||||
test_read_after_write_through_another_api_process (test_cache.CacheTests.test_read_after_write_through_another_api_process) ... ok
|
||||
test_version_rolls_back_with_failed_message (test_cache.CacheTests.test_version_rolls_back_with_failed_message) ... ok
|
||||
test_migrate_empty_schema_and_preserve_orm_data_on_repeat (test_database.MigrationTests.test_migrate_empty_schema_and_preserve_orm_data_on_repeat) ... ok
|
||||
test_chat_rollback (test_lab2.Lab2Tests.test_chat_rollback) ... ok
|
||||
test_expiry_short_ttl (test_lab2.Lab2Tests.test_expiry_short_ttl) ... ok
|
||||
test_migrations_repeat (test_lab2.Lab2Tests.test_migrations_repeat) ... ok
|
||||
test_password_logout_and_identity (test_lab2.Lab2Tests.test_password_logout_and_identity) ... ok
|
||||
test_registration_race (test_lab2.Lab2Tests.test_registration_race) ... ok
|
||||
test_unicode_password (test_lab2.Lab2Tests.test_unicode_password) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 13 tests in 18.089s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,9 @@
|
||||
Command: docker compose restart api postgres redis
|
||||
Before (UTC):
|
||||
/mtusi2026lab2-api-1 2026-10-09T12:45:16.929905363Z
|
||||
/mtusi2026lab2-postgres-1 2026-10-09T12:36:54.655202315Z
|
||||
/mtusi2026lab2-redis-1 2026-10-09T12:47:22.636577776Z
|
||||
After (UTC):
|
||||
/mtusi2026lab2-api-1 2026-10-09T12:49:18.471856288Z
|
||||
/mtusi2026lab2-postgres-1 2026-10-09T12:49:17.844464627Z
|
||||
/mtusi2026lab2-redis-1 2026-10-09T12:49:17.810209464Z
|
||||
@@ -0,0 +1,13 @@
|
||||
Command: python student-tests/failures.py
|
||||
Container mtusi2026lab2-redis-1 Stopping
|
||||
Container mtusi2026lab2-redis-1 Stopped
|
||||
PASS redis unavailable: 503; seconds=[3.043, 3.052, 3.035]; anonymous=401; live=200
|
||||
Container mtusi2026lab2-redis-1 Starting
|
||||
Container mtusi2026lab2-redis-1 Started
|
||||
PASS redis recovered in 6.13s: same session and message
|
||||
Container mtusi2026lab2-postgres-1 Stopping
|
||||
Container mtusi2026lab2-postgres-1 Stopped
|
||||
PASS postgres unavailable: 503; seconds=[2.038, 4.033, 4.027]; anonymous=401; live=200
|
||||
Container mtusi2026lab2-postgres-1 Starting
|
||||
Container mtusi2026lab2-postgres-1 Started
|
||||
PASS postgres recovered in 6.14s: same session and message
|
||||
@@ -0,0 +1,6 @@
|
||||
Command: docker compose exec -T api python tests/smoke.py
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
Command: .venv/Scripts/python.exe tests/resilience.py --mode persistence --base-url http://127.0.0.1:8080 --action-script scripts/restart.cmd
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,17 @@
|
||||
Command: docker compose exec -T api python -m unittest discover -s student-tests -v
|
||||
test_hit_still_checks_membership_and_cache_failure_falls_back (test_cache.CacheTests.test_hit_still_checks_membership_and_cache_failure_falls_back) ... ok
|
||||
test_http_read_after_write_and_cursor_pages (test_cache.CacheTests.test_http_read_after_write_and_cursor_pages) ... ok
|
||||
test_late_fill_cannot_restore_old_page (test_cache.CacheTests.test_late_fill_cannot_restore_old_page) ... ok
|
||||
test_read_after_write_through_another_api_process (test_cache.CacheTests.test_read_after_write_through_another_api_process) ... ok
|
||||
test_version_rolls_back_with_failed_message (test_cache.CacheTests.test_version_rolls_back_with_failed_message) ... ok
|
||||
test_chat_rollback (test_lab2.Lab2Tests.test_chat_rollback) ... ok
|
||||
test_expiry_short_ttl (test_lab2.Lab2Tests.test_expiry_short_ttl) ... ok
|
||||
test_migrations_repeat (test_lab2.Lab2Tests.test_migrations_repeat) ... ok
|
||||
test_password_logout_and_identity (test_lab2.Lab2Tests.test_password_logout_and_identity) ... ok
|
||||
test_registration_race (test_lab2.Lab2Tests.test_registration_race) ... ok
|
||||
test_unicode_password (test_lab2.Lab2Tests.test_unicode_password) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 11 tests in 13.089s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,25 @@
|
||||
# Схема и транзакции (план реализации)
|
||||
|
||||
- `users`: UUID PK, уникальный username, display_name, password_hash, created_at.
|
||||
- `chats`: UUID PK, title, created_at.
|
||||
- `chat_members`: составной PK (chat_id, user_id), FK на chats и users.
|
||||
- `messages`: UUID PK, FK chat_id и sender_id, text, created_at.
|
||||
- Индекс `messages(chat_id, created_at, id)` обслуживает историю:
|
||||
`WHERE chat_id = %s AND (created_at, id) > (%s, %s)
|
||||
ORDER BY created_at, id LIMIT %s`. Берём limit + 1 для next_cursor.
|
||||
- Индекс `chat_members(user_id, chat_id)` — список чатов пользователя.
|
||||
- Создание чата и всех участников — одна транзакция; ошибка любого FK
|
||||
откатывает также сам чат. Регистрация и запись сообщения — отдельные транзакции.
|
||||
- UNIQUE username разрешает гонку регистрации на стороне PostgreSQL.
|
||||
- Версии SQL-миграций записываются в `schema_migrations` в той же транзакции,
|
||||
что и DDL; advisory lock защищает одновременный запуск миграций.
|
||||
- Redis — источник истины для сессий: ключ с SHA-256 случайного токена,
|
||||
значение user_id и expires_at, абсолютный TTL. Logout удаляет ключ.
|
||||
AOF + appendfsync always и именованный volume сохраняют действующие сессии.
|
||||
При отказе Redis защищённый запрос получает 503, обхода аутентификации нет.
|
||||
- Трек 1: migration 002 добавляет `chats.history_version`. Запись сообщения
|
||||
увеличивает версию в той же транзакции (сначала UPDATE, затем INSERT).
|
||||
Redis хранит страницы по ключу chat_id + version + hash(limit, cursor), TTL 30 с.
|
||||
Членство и версия всегда читаются из PostgreSQL. Старые ключи удаляются по TTL;
|
||||
логическая инвалидация происходит атомарно при COMMIT версии и сообщения.
|
||||
Опоздавшее заполнение старого ключа не влияет на новую версию.
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0;
|
||||
@@ -11,3 +11,7 @@ idna==3.20
|
||||
pydantic_core==2.46.5
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
psycopg[binary]==3.2.6
|
||||
SQLAlchemy==2.0.41
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Track 1: deterministic stale-fill race and measured reads on real PostgreSQL/Redis.
|
||||
|
||||
The unsafe cache exists ONLY in this experiment; API always uses versioned keys.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import resource
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, event, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from src.auth import sessions
|
||||
from src.cache import HistoryCache
|
||||
from src.database import create_db_engine
|
||||
from src.models import User, Chat, ChatMember
|
||||
from src.store import PostgresStore
|
||||
from .fixtures import seed_messages
|
||||
|
||||
|
||||
class NaiveCache(HistoryCache):
|
||||
def key(self, chat_id, version, limit, after):
|
||||
return super().key(chat_id, 0, limit, after)
|
||||
|
||||
|
||||
class NoCache:
|
||||
def get_or_load(self, chat_id, version, limit, after, load):
|
||||
return load()
|
||||
|
||||
|
||||
class QueryCounter:
|
||||
lock = threading.Lock()
|
||||
counts = {'sql': 0, 'history_selects': 0}
|
||||
|
||||
@classmethod
|
||||
def record(cls, conn, cursor, statement, parameters, context, executemany):
|
||||
with cls.lock:
|
||||
cls.counts['sql'] += 1
|
||||
normalized = ' '.join(statement.split()).lower()
|
||||
if normalized.startswith('select ') and ' from messages ' in normalized:
|
||||
cls.counts['history_selects'] += 1
|
||||
|
||||
@classmethod
|
||||
def reset(cls):
|
||||
with cls.lock:
|
||||
cls.counts = {'sql': 0, 'history_selects': 0}
|
||||
|
||||
|
||||
def make_store(engine, cache):
|
||||
store = PostgresStore(cache)
|
||||
store.sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
return store
|
||||
|
||||
|
||||
def stale_race(reader, writer, actor_id, chat_id, naive=False):
|
||||
"""Pause AFTER the real history SELECT, commit a write, then finish cache SET."""
|
||||
loaded, resume = threading.Event(), threading.Event()
|
||||
original_load = reader._load_messages
|
||||
|
||||
def paused_load(*args):
|
||||
page = original_load(*args)
|
||||
loaded.set()
|
||||
if not resume.wait(10):
|
||||
raise AssertionError('Race writer did not finish')
|
||||
return page
|
||||
|
||||
reader._load_messages = paused_load
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(reader.list_messages, chat_id, actor_id, 100, None)
|
||||
try:
|
||||
assert loaded.wait(10), 'Reader did not reach the cache-fill barrier'
|
||||
message = writer.create_message(chat_id, actor_id, 'committed during cache fill')
|
||||
if naive:
|
||||
reader.cache.client.delete(reader.cache.key(chat_id, 0, 100, None))
|
||||
finally:
|
||||
resume.set()
|
||||
overlapping_page = pending.result(timeout=10)
|
||||
finally:
|
||||
reader._load_messages = original_load
|
||||
assert message['id'] not in {item['id'] for item in overlapping_page['items']}
|
||||
started = time.perf_counter()
|
||||
page = reader.list_messages(chat_id, actor_id, 100, None)
|
||||
return {'stale': message['id'] not in {item['id'] for item in page['items']},
|
||||
'read_after_write_ms': round((time.perf_counter() - started) * 1000, 3)}
|
||||
|
||||
|
||||
def benchmark(store, actor_id, chat_id, requests=300, concurrency=8):
|
||||
if isinstance(store.cache, HistoryCache):
|
||||
for key in store.cache.client.scan_iter(match=f'{store.cache.prefix}:{chat_id}:*'):
|
||||
store.cache.client.delete(key)
|
||||
expected = store.list_messages(chat_id, actor_id, 100, None)
|
||||
QueryCounter.reset()
|
||||
started = time.perf_counter()
|
||||
cpu_started = time.process_time()
|
||||
|
||||
def read(_):
|
||||
tick = time.perf_counter()
|
||||
page = store.list_messages(chat_id, actor_id, 100, None)
|
||||
assert page == expected
|
||||
return (time.perf_counter() - tick) * 1000
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
latency = sorted(executor.map(read, range(requests)))
|
||||
elapsed = time.perf_counter() - started
|
||||
result = {
|
||||
'requests': requests, 'concurrency': concurrency, 'errors': 0,
|
||||
'seconds': round(elapsed, 3), 'successful_rps': round(requests / elapsed, 2),
|
||||
'client_cpu_seconds': round(time.process_time() - cpu_started, 3),
|
||||
'client_peak_rss_mib': round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 2),
|
||||
**QueryCounter.counts,
|
||||
}
|
||||
for percentile in (50, 95, 99):
|
||||
result[f'p{percentile}_ms'] = round(latency[math.ceil(len(latency) * percentile / 100) - 1], 3)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
prefix = 'experiment:' + uuid4().hex
|
||||
engine = create_db_engine()
|
||||
event.listen(engine, 'before_cursor_execute', QueryCounter.record)
|
||||
writer = make_store(engine, NoCache())
|
||||
user = writer.create_user('exp_' + uuid4().hex[:12], 'Experiment', 'Synthetic-password-123')
|
||||
result = {'race_repeats': 5, 'race': {}, 'benchmark': {}, 'messages': 10000,
|
||||
'page_size': 100, 'warmup_reads_per_run': 1,
|
||||
'scope': 'store calls; real SQL and Redis; excludes HTTP, sessions and KDF; SQL excludes BEGIN/COMMIT'}
|
||||
try:
|
||||
for name, cache_type in [('naive', NaiveCache), ('versioned', HistoryCache)]:
|
||||
results = []
|
||||
for n in range(5):
|
||||
chat = writer.create_chat(user['id'], 'Race', [user['id']])
|
||||
reader = make_store(engine, cache_type(sessions.client, prefix=f'{prefix}:{name}:{n}'))
|
||||
QueryCounter.reset()
|
||||
row = stale_race(reader, writer, user['id'], chat['id'], naive=name == 'naive')
|
||||
row.update(QueryCounter.counts)
|
||||
assert row['stale'] == (name == 'naive')
|
||||
results.append(row)
|
||||
result['race'][name] = results
|
||||
|
||||
chat = writer.create_chat(user['id'], 'Benchmark', [user['id']])
|
||||
with writer.sessions.begin() as session:
|
||||
seed_messages(session, chat['id'], user['id'])
|
||||
modes = {
|
||||
'uncached': make_store(engine, NoCache()),
|
||||
'naive': make_store(engine, NaiveCache(sessions.client, prefix=prefix + ':bench-naive')),
|
||||
'versioned': make_store(engine, HistoryCache(sessions.client, prefix=prefix + ':bench-versioned')),
|
||||
}
|
||||
result['benchmark'] = {name: [] for name in modes}
|
||||
names = list(modes)
|
||||
for repeat in range(3):
|
||||
for name in names[repeat:] + names[:repeat]:
|
||||
result['benchmark'][name].append(benchmark(modes[name], user['id'], chat['id']))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
for key in sessions.client.scan_iter(match=prefix + ':*'):
|
||||
sessions.client.delete(key)
|
||||
with writer.sessions.begin() as session:
|
||||
session.execute(delete(Chat).where(Chat.id.in_(
|
||||
select(ChatMember.chat_id).where(ChatMember.user_id == user['id']))))
|
||||
session.execute(delete(User).where(User.id == user['id']))
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""EXPLAIN the actual ORM history query on 10,000 rows, then roll back fixtures."""
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.database import create_db_engine
|
||||
from src.models import User, Chat, Message
|
||||
from src.store import history_query
|
||||
from .fixtures import seed_messages
|
||||
|
||||
|
||||
def main():
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
user = User(username='explain_' + uuid4().hex[:12], display_name='Explain', password_hash='unused')
|
||||
session.add(user)
|
||||
session.flush()
|
||||
chat = Chat(title='Explain')
|
||||
session.add(chat)
|
||||
session.flush()
|
||||
seed_messages(session, chat.id, user.id)
|
||||
conn = session.connection()
|
||||
conn.exec_driver_sql('ANALYZE messages')
|
||||
boundary = session.execute(select(Message.created_at, Message.id).where(
|
||||
Message.chat_id == chat.id).order_by(Message.created_at, Message.id).offset(8999).limit(1)).one()
|
||||
after = (boundary.created_at.isoformat(), boundary.id)
|
||||
compiled = history_query(chat.id, 50, after).compile(dialect=engine.dialect)
|
||||
print('Dataset: 10000 messages; page after message 9000; ORM query; limit 51')
|
||||
for row in conn.exec_driver_sql('EXPLAIN (ANALYZE, BUFFERS) ' + str(compiled), compiled.params):
|
||||
print(row[0])
|
||||
session.rollback()
|
||||
print('Synthetic data rolled back.')
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Synthetic message history shared by EXPLAIN and cache measurements."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import insert, update
|
||||
|
||||
from src.models import Chat, Message
|
||||
|
||||
|
||||
def seed_messages(session, chat_id, user_id, count=10000):
|
||||
start = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
session.execute(insert(Message), [
|
||||
{'chat_id': chat_id, 'sender_id': user_id, 'text': 'synthetic ' * 100,
|
||||
'created_at': start + timedelta(milliseconds=n)}
|
||||
for n in range(1, count + 1)
|
||||
])
|
||||
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||
history_version=Chat.history_version + 1))
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\restart.py
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Restart the whole lab stack, retaining volumes and recording actual start times."""
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
|
||||
return subprocess.check_output(
|
||||
['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids],
|
||||
cwd=ROOT, text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
before = snapshot()
|
||||
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
|
||||
cwd=ROOT, check=True, timeout=60)
|
||||
after = snapshot()
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
(ROOT / 'evidence' / 'restart.txt').write_text(
|
||||
'Command: docker compose restart api postgres redis\nBefore (UTC):\n' + before
|
||||
+ 'After (UTC):\n' + after, encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/restart.py
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Run on the host: research measurements and a real Redis restart through HTTP."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8', timeout=180)
|
||||
|
||||
|
||||
def cache_state(chat_id, clear=False):
|
||||
code = """import json, sys
|
||||
from uuid import UUID
|
||||
from src.auth import sessions
|
||||
keys = list(sessions.client.scan_iter(match='history:' + str(UUID(sys.argv[1])) + ':*'))
|
||||
print(json.dumps({'keys': len(keys)}))
|
||||
if sys.argv[2] == 'clear' and keys:
|
||||
sessions.client.delete(*keys)
|
||||
"""
|
||||
return json.loads(command('docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code,
|
||||
chat_id, 'clear' if clear else 'inspect'))
|
||||
|
||||
|
||||
def restart_check():
|
||||
client = Client('http://127.0.0.1:8080')
|
||||
actor = client.user('restartcache')
|
||||
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
||||
body={'title': 'Cache restart', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
def read():
|
||||
return client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
first = client.message(actor, chat, 'before Redis restart')['json']
|
||||
assert read()['items'] == [first]
|
||||
cached_before = cache_state(chat['id'])['keys']
|
||||
assert cached_before > 0
|
||||
second = client.message(actor, chat, 'committed after cache fill')['json']
|
||||
redis_id = command('docker', 'compose', 'ps', '-q', 'redis').strip()
|
||||
before_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip()
|
||||
started = time.monotonic()
|
||||
command('docker', 'compose', 'restart', 'redis')
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call('GET', '/health/ready')
|
||||
break
|
||||
except AssertionError:
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise AssertionError('Redis recovery timed out')
|
||||
after_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip()
|
||||
assert before_start != after_start
|
||||
assert read()['items'] == [first, second]
|
||||
recovery = round(time.monotonic() - started, 3)
|
||||
cache_state(chat['id'], clear=True)
|
||||
assert cache_state(chat['id'])['keys'] == 0
|
||||
assert read()['items'] == [first, second]
|
||||
assert cache_state(chat['id'])['keys'] > 0
|
||||
third = client.message(actor, chat, 'after Redis recovery')['json']
|
||||
assert read()['items'] == [first, second, third]
|
||||
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
client.call('GET', path, 401, actor=actor)
|
||||
return {'status': 'PASS', 'same_session': True, 'confirmed_messages': 3,
|
||||
'cold_cache_rebuilt': True, 'read_after_write': True, 'logout_still_401': True,
|
||||
'recovery_seconds': recovery, 'redis_start_before_utc': before_start,
|
||||
'redis_start_after_utc': after_start}
|
||||
|
||||
|
||||
def main():
|
||||
evidence = ROOT / 'evidence'
|
||||
evidence.mkdir(exist_ok=True)
|
||||
result = json.loads(command('docker', 'compose', 'exec', '-T', 'api',
|
||||
'python', '-m', 'scripts.cache_experiment'))
|
||||
(evidence / 'cache-experiment.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
||||
result = restart_check()
|
||||
(evidence / 'cache-restart.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
||||
ids = command('docker', 'compose', 'ps', '-q').split()
|
||||
snapshot = command('docker', 'stats', '--no-stream', '--format', '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}', *ids)
|
||||
(evidence / 'cache-resources.txt').write_text('Post-experiment snapshot (not peak):\n' + snapshot, encoding='utf-8')
|
||||
print('PASS: stale race reproduced before protection and prevented after; SQL/latency measured.')
|
||||
print('PASS: Redis restart, same session, read-after-write, cold cache rebuild, logout.')
|
||||
print('Evidence: cache-experiment.json, cache-restart.json, cache-resources.txt')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -9,7 +9,6 @@ root = Path(__file__).resolve().parents[1]
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text())
|
||||
validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from redis import Redis
|
||||
|
||||
from .errors import APIError
|
||||
|
||||
password_hasher = PasswordHasher()
|
||||
dummy_hash = password_hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
|
||||
def verify_password(encoded, password):
|
||||
try:
|
||||
return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None
|
||||
except VerificationError:
|
||||
return False
|
||||
|
||||
|
||||
def unauthorized():
|
||||
return APIError(401, 'unauthorized', 'Нет действительной идентификации')
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self):
|
||||
self.client = Redis.from_url(
|
||||
os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=1, socket_timeout=1, max_connections=20,
|
||||
retry_on_timeout=False,
|
||||
)
|
||||
self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
|
||||
if self.ttl < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
|
||||
@staticmethod
|
||||
def key(token):
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise unauthorized()
|
||||
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
|
||||
|
||||
def create(self, user):
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl)
|
||||
expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}),
|
||||
pxat=int(expires.timestamp() * 1000))
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user}
|
||||
|
||||
def user_id(self, token):
|
||||
raw = self.client.get(self.key(token))
|
||||
if raw is None:
|
||||
raise unauthorized()
|
||||
session = json.loads(raw)
|
||||
if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc):
|
||||
raise unauthorized()
|
||||
return session['user_id']
|
||||
|
||||
def delete(self, token):
|
||||
if not self.client.delete(self.key(token)):
|
||||
raise unauthorized()
|
||||
|
||||
|
||||
sessions = Sessions()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""History pages are disposable; PostgreSQL owns both data and their version."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .auth import sessions
|
||||
|
||||
|
||||
class HistoryCache:
|
||||
def __init__(self, client, ttl=30, prefix='history'):
|
||||
self.client = client
|
||||
self.ttl = ttl
|
||||
self.prefix = prefix
|
||||
if ttl < 1:
|
||||
raise ValueError('CACHE_TTL_SECONDS must be positive')
|
||||
|
||||
def key(self, chat_id, version, limit, after):
|
||||
page = json.dumps([limit, after], separators=(',', ':'))
|
||||
digest = hashlib.sha256(page.encode()).hexdigest()
|
||||
return f'{self.prefix}:{chat_id}:{version}:{digest}'
|
||||
|
||||
def get_or_load(self, chat_id, version, limit, after, load):
|
||||
key = self.key(chat_id, version, limit, after)
|
||||
try:
|
||||
cached = self.client.get(key)
|
||||
except RedisError:
|
||||
return load()
|
||||
if cached is not None:
|
||||
return json.loads(cached)
|
||||
page = load()
|
||||
try:
|
||||
self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl)
|
||||
except RedisError:
|
||||
pass
|
||||
return page
|
||||
|
||||
|
||||
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')))
|
||||
@@ -0,0 +1,51 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, func, insert, select
|
||||
from sqlalchemy.engine import URL, make_url
|
||||
|
||||
from .models import SchemaMigration
|
||||
|
||||
|
||||
def database_url():
|
||||
if os.getenv('DATABASE_URL'):
|
||||
url = make_url(os.environ['DATABASE_URL'])
|
||||
if url.drivername in ('postgres', 'postgresql'):
|
||||
url = url.set(drivername='postgresql+psycopg')
|
||||
if url.drivername != 'postgresql+psycopg':
|
||||
raise ValueError('DATABASE_URL must use PostgreSQL with psycopg')
|
||||
return url
|
||||
return URL.create(
|
||||
'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'),
|
||||
database=os.getenv('POSTGRES_DB', 'messenger'),
|
||||
username=os.getenv('POSTGRES_USER', 'messenger'),
|
||||
password=os.environ['POSTGRES_PASSWORD'],
|
||||
)
|
||||
|
||||
|
||||
def create_db_engine():
|
||||
return create_engine(
|
||||
database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True,
|
||||
connect_args={'connect_timeout': 2,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000'},
|
||||
)
|
||||
|
||||
|
||||
def migrate(engine):
|
||||
with engine.begin() as conn:
|
||||
conn.execute(select(func.pg_advisory_xact_lock(202602)))
|
||||
SchemaMigration.__table__.create(conn, checkfirst=True)
|
||||
applied = set(conn.scalars(select(SchemaMigration.version)))
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||
if path.name not in applied:
|
||||
conn.exec_driver_sql(path.read_text(encoding='utf-8'))
|
||||
conn.execute(insert(SchemaMigration).values(version=path.name))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
migrate(engine)
|
||||
print('Migrations applied')
|
||||
finally:
|
||||
engine.dispose()
|
||||
+14
-10
@@ -1,7 +1,8 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Header, Query
|
||||
from fastapi import Depends, Header, Query
|
||||
|
||||
from .auth import sessions, unauthorized
|
||||
from .errors import APIError
|
||||
from .store import store
|
||||
|
||||
@@ -16,12 +17,15 @@ def page_limit(limit: str = Query(default="50")) -> int:
|
||||
|
||||
|
||||
|
||||
def current_user(x_user_id: Optional[str] = Header(default=None, alias="X-User-Id")) -> dict:
|
||||
if not x_user_id:
|
||||
raise APIError(401, "unauthorized", "Нет действительной идентификации")
|
||||
try:
|
||||
return store.require_user(x_user_id)
|
||||
except APIError as exc:
|
||||
if exc.status_code == 400:
|
||||
raise APIError(401, "unauthorized", "Нет действительной идентификации") from exc
|
||||
raise
|
||||
def bearer_token(authorization: Optional[str] = Header(default=None)) -> str:
|
||||
parts = (authorization or '').split()
|
||||
if len(parts) != 2 or parts[0].lower() != 'bearer':
|
||||
raise unauthorized()
|
||||
return parts[1]
|
||||
|
||||
|
||||
def current_user(token: str = Depends(bearer_token)) -> dict:
|
||||
user = store.get_user(sessions.user_id(token))
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
@@ -30,9 +30,18 @@ def api_error_handler(request: Request, exc: APIError) -> JSONResponse:
|
||||
response = error_response(request, exc.status_code, exc.code, exc.message)
|
||||
if exc.status_code == 429:
|
||||
response.headers['Retry-After'] = '1'
|
||||
if exc.status_code == 401:
|
||||
response.headers['WWW-Authenticate'] = (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
|
||||
|
||||
|
||||
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return error_response(request, 400, "invalid_request", "Неверный запрос")
|
||||
|
||||
|
||||
+27
-2
@@ -1,22 +1,47 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from starlette.exceptions import HTTPException
|
||||
from contextlib import asynccontextmanager
|
||||
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from redis.exceptions import RedisError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler
|
||||
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
|
||||
from .database import create_db_engine, migrate
|
||||
from .store import store
|
||||
from .auth import sessions
|
||||
from .middleware import RequestIdMiddleware
|
||||
from .limits import ResourceLimitMiddleware
|
||||
from .routers.health import router as health_router
|
||||
from .routers.users import router as users_router
|
||||
from .routers.chats import router as chats_router
|
||||
from .routers.auth import router as auth_router
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
await run_in_threadpool(migrate, engine)
|
||||
store.sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
yield
|
||||
finally:
|
||||
engine.dispose()
|
||||
sessions.client.close()
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger", lifespan=lifespan)
|
||||
app.add_middleware(ResourceLimitMiddleware)
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
app.add_exception_handler(APIError, api_error_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||
app.add_exception_handler(HTTPException, http_error_handler)
|
||||
app.add_exception_handler(Exception, internal_error_handler)
|
||||
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError):
|
||||
app.add_exception_handler(error, dependency_error_handler)
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(users_router)
|
||||
app.include_router(chats_router)
|
||||
app.include_router(auth_router)
|
||||
|
||||
+3
-1
@@ -9,7 +9,9 @@ class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
|
||||
if request.method in {"POST", "PUT", "PATCH"} and request.url.path.startswith("/api/"):
|
||||
if (request.method in {"POST", "PUT", "PATCH"}
|
||||
and request.url.path.startswith("/api/")
|
||||
and request.url.path != '/api/v1/auth/sessions'):
|
||||
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if content_type != "application/json":
|
||||
response = JSONResponse(
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'users'
|
||||
__table_args__ = (
|
||||
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"),
|
||||
CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
username: Mapped[str] = mapped_column(Text, unique=True)
|
||||
display_name: Mapped[str] = mapped_column(Text)
|
||||
password_hash: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = 'chats'
|
||||
__table_args__ = (
|
||||
CheckConstraint('char_length(title) BETWEEN 1 AND 100'),
|
||||
Index('chats_order_idx', 'created_at', 'id'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
history_version: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = 'chat_members'
|
||||
__table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),)
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = 'messages'
|
||||
__table_args__ = (
|
||||
CheckConstraint('char_length(text) BETWEEN 1 AND 4000'),
|
||||
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
class SchemaMigration(Base):
|
||||
__tablename__ = 'schema_migrations'
|
||||
version: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
|
||||
@@ -0,0 +1,34 @@
|
||||
import base64
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Request, Response
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from ..auth import sessions, unauthorized
|
||||
from ..dependencies import bearer_token, current_user
|
||||
from ..errors import APIError
|
||||
from ..store import store
|
||||
|
||||
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||
|
||||
|
||||
@router.post('', status_code=201)
|
||||
async def create_session(request: Request, authorization: str = Header(default='')):
|
||||
if await request.body():
|
||||
raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса')
|
||||
try:
|
||||
scheme, encoded = authorization.split()
|
||||
if scheme.lower() != 'basic':
|
||||
raise ValueError()
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username:
|
||||
raise ValueError()
|
||||
except (ValueError, UnicodeError):
|
||||
raise unauthorized() from None
|
||||
user = await run_in_threadpool(store.login, username, password)
|
||||
return await run_in_threadpool(sessions.create, user)
|
||||
|
||||
|
||||
@router.delete('/current', status_code=204)
|
||||
def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)):
|
||||
sessions.delete(token)
|
||||
return Response(status_code=204)
|
||||
@@ -1,6 +1,9 @@
|
||||
from fastapi import APIRouter
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..schemas.common import Health
|
||||
from ..auth import sessions
|
||||
from ..store import store
|
||||
|
||||
router = APIRouter(prefix='/health')
|
||||
|
||||
@@ -12,4 +15,7 @@ def health_live():
|
||||
|
||||
@router.get("/ready", response_model=Health)
|
||||
def health_ready():
|
||||
with store.sessions() as session:
|
||||
session.execute(select(1))
|
||||
sessions.client.ping()
|
||||
return {"status": "ok"}
|
||||
|
||||
@@ -11,7 +11,7 @@ router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
@router.post("", response_model=User, status_code=status.HTTP_201_CREATED)
|
||||
def create_user(payload: CreateUser):
|
||||
return store.create_user(payload.username, payload.display_name)
|
||||
return store.create_user(payload.username, payload.display_name, payload.password)
|
||||
|
||||
|
||||
@router.get("/me", response_model=User)
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from pydantic import BaseModel, ConfigDict, field_validator
|
||||
|
||||
|
||||
class ContractModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator('*')
|
||||
def validate_text_encoding(cls, value, info):
|
||||
if isinstance(value, str):
|
||||
value.encode('utf-8')
|
||||
if info.field_name != 'password' and '\x00' in value:
|
||||
raise ValueError('NUL is not supported in PostgreSQL text')
|
||||
return value
|
||||
|
||||
|
||||
class Health(ContractModel):
|
||||
status: str
|
||||
|
||||
@@ -12,6 +12,7 @@ USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
class CreateUser(ContractModel):
|
||||
username: StrictStr
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
def validate_username(cls, value):
|
||||
|
||||
+139
-150
@@ -1,180 +1,169 @@
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from threading import RLock
|
||||
from typing import Optional, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
from typing import Tuple
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from .auth import password_hasher, verify_password, unauthorized
|
||||
from .errors import APIError
|
||||
from .limits import limits
|
||||
from .cache import history_cache
|
||||
from .models import User, Chat, ChatMember, Message
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def ensure_uuid(value: str) -> str:
|
||||
def ensure_uuid(value):
|
||||
try:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("UUID must be a string")
|
||||
raise ValueError()
|
||||
return str(UUID(value))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise APIError(400, "invalid_request", "Неверный UUID") from exc
|
||||
raise APIError(400, 'invalid_request', 'Неверный UUID') from exc
|
||||
|
||||
|
||||
class InMemoryStore:
|
||||
def __init__(self, settings=limits):
|
||||
self.settings = settings
|
||||
self._lock = RLock()
|
||||
self._users = {}
|
||||
self._usernames = set()
|
||||
self._chats = {}
|
||||
self._messages = {}
|
||||
self._message_count = 0
|
||||
self._text_bytes = 0
|
||||
self._tokens = float(settings.message_burst)
|
||||
self._refilled_at = time.monotonic()
|
||||
def serialize(model, *fields):
|
||||
if model is None:
|
||||
return None
|
||||
result = {}
|
||||
for field in fields:
|
||||
value = getattr(model, field)
|
||||
if isinstance(value, datetime):
|
||||
value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
result[field] = value
|
||||
return result
|
||||
|
||||
def create_user(self, username: str, display_name: str) -> dict:
|
||||
with self._lock:
|
||||
if username in self._usernames:
|
||||
raise APIError(409, "conflict", "Имя пользователя уже занято")
|
||||
if len(self._users) >= self.settings.max_users:
|
||||
raise APIError(503, 'unavailable', 'Достигнут лимит пользователей')
|
||||
|
||||
user = {
|
||||
"id": str(uuid4()),
|
||||
"username": username,
|
||||
"display_name": display_name,
|
||||
"created_at": utc_now(),
|
||||
}
|
||||
self._users[user["id"]] = user
|
||||
self._usernames.add(username)
|
||||
return dict(user)
|
||||
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
|
||||
CHAT_FIELDS = ('id', 'title', 'created_at')
|
||||
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at')
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[dict]:
|
||||
user_id = ensure_uuid(user_id)
|
||||
with self._lock:
|
||||
user = self._users.get(user_id)
|
||||
return dict(user) if user else None
|
||||
|
||||
def require_user(self, user_id: str) -> dict:
|
||||
user = self.get_user(user_id)
|
||||
if not user:
|
||||
raise APIError(401, "unauthorized", "Пользователь не авторизован")
|
||||
return user
|
||||
def after_condition(model, after):
|
||||
return tuple_(model.created_at, model.id) > (
|
||||
datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1])
|
||||
|
||||
def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict:
|
||||
actor_id = ensure_uuid(actor_id)
|
||||
normalized_members = [ensure_uuid(member_id) for member_id in member_ids]
|
||||
if len(normalized_members) != len(set(normalized_members)):
|
||||
raise APIError(400, "invalid_request", "Участники не должны повторяться")
|
||||
if actor_id not in normalized_members:
|
||||
raise APIError(400, "invalid_request", "Создатель должен быть участником чата")
|
||||
|
||||
with self._lock:
|
||||
if any(member_id not in self._users for member_id in normalized_members):
|
||||
raise APIError(400, "invalid_request", "Неизвестный участник")
|
||||
if len(self._chats) >= self.settings.max_chats:
|
||||
raise APIError(503, 'unavailable', 'Достигнут лимит чатов')
|
||||
def history_query(chat_id, limit, after):
|
||||
query = select(Message).where(Message.chat_id == chat_id)
|
||||
if after:
|
||||
query = query.where(after_condition(Message, after))
|
||||
return query.order_by(Message.created_at, Message.id).limit(limit + 1)
|
||||
|
||||
chat = {
|
||||
"id": str(uuid4()),
|
||||
"title": title,
|
||||
"member_ids": normalized_members,
|
||||
"created_at": utc_now(),
|
||||
}
|
||||
self._chats[chat["id"]] = chat
|
||||
self._messages[chat["id"]] = []
|
||||
return self._copy_chat(chat)
|
||||
|
||||
def get_chat_for_user(self, chat_id: str, actor_id: str) -> dict:
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
actor_id = ensure_uuid(actor_id)
|
||||
with self._lock:
|
||||
chat = self._chats.get(chat_id)
|
||||
if not chat or actor_id not in chat["member_ids"]:
|
||||
raise APIError(404, "not_found", "Объект не найден")
|
||||
return self._copy_chat(chat)
|
||||
class PostgresStore:
|
||||
def __init__(self, cache=history_cache):
|
||||
self.sessions = None
|
||||
self.cache = cache
|
||||
|
||||
def list_chats(self, actor_id: str, limit: int, cursor: Optional[str]) -> dict:
|
||||
actor_id = ensure_uuid(actor_id)
|
||||
after = self._decode_cursor(cursor, "chats", actor_id) if cursor else None
|
||||
|
||||
with self._lock:
|
||||
chats = [
|
||||
self._copy_chat(chat)
|
||||
for chat in self._chats.values()
|
||||
if actor_id in chat["member_ids"]
|
||||
]
|
||||
chats.sort(key=lambda item: (item["created_at"], item["id"]))
|
||||
if after:
|
||||
chats = [chat for chat in chats if (chat["created_at"], chat["id"]) > after]
|
||||
|
||||
page = chats[:limit]
|
||||
next_cursor = None
|
||||
if len(chats) > limit and page:
|
||||
last = page[-1]
|
||||
next_cursor = self._encode_cursor("chats", actor_id, last["created_at"], last["id"])
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
|
||||
def create_message(self, chat_id: str, actor_id: str, text: str) -> dict:
|
||||
chat = self.get_chat_for_user(chat_id, actor_id)
|
||||
def create_user(self, username, display_name, password):
|
||||
user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password))
|
||||
try:
|
||||
text_size = len(text.encode('utf-8'))
|
||||
except UnicodeEncodeError as exc:
|
||||
raise APIError(400, 'invalid_request', 'Некорректный Unicode') from exc
|
||||
with self._lock:
|
||||
if self._message_count >= self.settings.max_messages:
|
||||
raise APIError(503, 'unavailable', 'Достигнут лимит сообщений')
|
||||
if self._text_bytes + text_size > self.settings.max_text_bytes:
|
||||
raise APIError(503, 'unavailable', 'Достигнут лимит объёма текста')
|
||||
now = time.monotonic()
|
||||
self._tokens = min(self.settings.message_burst,
|
||||
self._tokens + (now - self._refilled_at) * self.settings.message_rate)
|
||||
self._refilled_at = now
|
||||
if self._tokens < 1:
|
||||
raise APIError(429, 'rate_limited', 'Слишком частая отправка сообщений')
|
||||
self._tokens -= 1
|
||||
message = {
|
||||
"id": str(uuid4()),
|
||||
"chat_id": chat["id"],
|
||||
"sender_id": ensure_uuid(actor_id),
|
||||
"text": text,
|
||||
"created_at": utc_now(),
|
||||
}
|
||||
self._messages[chat["id"]].append(message)
|
||||
self._message_count += 1
|
||||
self._text_bytes += text_size
|
||||
return dict(message)
|
||||
with self.sessions.begin() as session:
|
||||
session.add(user)
|
||||
session.flush()
|
||||
result = serialize(user, *USER_FIELDS)
|
||||
return result
|
||||
except IntegrityError as exc:
|
||||
if getattr(exc.orig, 'sqlstate', None) == '23505':
|
||||
raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc
|
||||
raise
|
||||
|
||||
def list_messages(self, chat_id: str, actor_id: str, limit: int, cursor: Optional[str]) -> dict:
|
||||
chat = self.get_chat_for_user(chat_id, actor_id)
|
||||
after = self._decode_cursor(cursor, "messages", chat["id"]) if cursor else None
|
||||
def get_user(self, user_id):
|
||||
with self.sessions() as session:
|
||||
return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS)
|
||||
|
||||
with self._lock:
|
||||
messages = [dict(message) for message in self._messages.get(chat["id"], [])]
|
||||
messages.sort(key=lambda item: (item["created_at"], item["id"]))
|
||||
if after:
|
||||
messages = [
|
||||
message
|
||||
for message in messages
|
||||
if (message["created_at"], message["id"]) > after
|
||||
]
|
||||
def login(self, username, password):
|
||||
with self.sessions() as session:
|
||||
user = session.scalar(select(User).where(User.username == username))
|
||||
encoded = user.password_hash if user else None
|
||||
result = serialize(user, *USER_FIELDS)
|
||||
if not verify_password(encoded, password):
|
||||
raise unauthorized()
|
||||
return result
|
||||
|
||||
page = messages[:limit]
|
||||
next_cursor = None
|
||||
if len(messages) > limit and page:
|
||||
last = page[-1]
|
||||
next_cursor = self._encode_cursor("messages", chat["id"], last["created_at"], last["id"])
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
def create_chat(self, actor_id, title, member_ids):
|
||||
members = [ensure_uuid(member) for member in member_ids]
|
||||
if len(members) != len(set(members)) or actor_id not in members:
|
||||
raise APIError(400, 'invalid_request', 'Неверный список участников')
|
||||
try:
|
||||
with self.sessions.begin() as session:
|
||||
chat = Chat(title=title)
|
||||
session.add(chat)
|
||||
session.flush()
|
||||
session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members])
|
||||
result = serialize(chat, *CHAT_FIELDS)
|
||||
result['member_ids'] = sorted(members)
|
||||
return result
|
||||
except IntegrityError as exc:
|
||||
if getattr(exc.orig, 'sqlstate', None) == '23503':
|
||||
raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _copy_chat(chat: dict) -> dict:
|
||||
copy = dict(chat)
|
||||
copy["member_ids"] = list(chat["member_ids"])
|
||||
return copy
|
||||
def _chat(session, chat_id, actor_id):
|
||||
chat = session.scalar(select(Chat).join(ChatMember).where(
|
||||
Chat.id == chat_id, ChatMember.user_id == actor_id))
|
||||
if chat is None:
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
return chat
|
||||
|
||||
@staticmethod
|
||||
def _members(session, chat):
|
||||
result = serialize(chat, *CHAT_FIELDS)
|
||||
result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where(
|
||||
ChatMember.chat_id == chat.id).order_by(ChatMember.user_id)))
|
||||
return result
|
||||
|
||||
def get_chat_for_user(self, chat_id, actor_id):
|
||||
with self.sessions() as session:
|
||||
return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id))
|
||||
|
||||
def list_chats(self, actor_id, limit, cursor):
|
||||
after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None
|
||||
query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id)
|
||||
if after:
|
||||
query = query.where(after_condition(Chat, after))
|
||||
query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1)
|
||||
with self.sessions() as session:
|
||||
chats = list(session.scalars(query))
|
||||
items = [self._members(session, chat) for chat in chats[:limit]]
|
||||
return self._page(items, len(chats) > limit, 'chats', actor_id)
|
||||
|
||||
def create_message(self, chat_id, actor_id, text):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
with self.sessions.begin() as session:
|
||||
self._chat(session, chat_id, actor_id)
|
||||
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||
history_version=Chat.history_version + 1))
|
||||
message = Message(chat_id=chat_id, sender_id=actor_id, text=text)
|
||||
session.add(message)
|
||||
session.flush()
|
||||
result = serialize(message, *MESSAGE_FIELDS)
|
||||
return result
|
||||
|
||||
def list_messages(self, chat_id, actor_id, limit, cursor):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None
|
||||
with self.sessions() as session:
|
||||
version = self._chat(session, chat_id, actor_id).history_version
|
||||
return self.cache.get_or_load(
|
||||
chat_id, version, limit, after,
|
||||
lambda: self._load_messages(chat_id, limit, after),
|
||||
)
|
||||
|
||||
def _load_messages(self, chat_id, limit, after):
|
||||
with self.sessions() as session:
|
||||
messages = list(session.scalars(history_query(chat_id, limit, after)))
|
||||
items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]]
|
||||
return self._page(items, len(messages) > limit, 'messages', chat_id)
|
||||
|
||||
def _page(self, items, has_more, resource, owner_id):
|
||||
cursor = None
|
||||
if has_more:
|
||||
last = items[-1]
|
||||
cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id'])
|
||||
return {'items': items, 'next_cursor': cursor}
|
||||
|
||||
@staticmethod
|
||||
def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str:
|
||||
@@ -215,4 +204,4 @@ class InMemoryStore:
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
store = InMemoryStore()
|
||||
store = PostgresStore()
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Explicitly stop each storage service on the local lab stack, then restore it."""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, timeout=60)
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(timeout=6)
|
||||
actor = client.user('failure')
|
||||
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
||||
body={'title': 'Failure test', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
||||
message = client.message(actor, chat, 'Must survive storage failure')['json']
|
||||
for service in ('redis', 'postgres'):
|
||||
try:
|
||||
compose('stop', service)
|
||||
elapsed = []
|
||||
for _ in range(3):
|
||||
result = client.call('GET', '/api/v1/users/me', 503, actor=actor)
|
||||
assert result['elapsed'] < 5, 'Dependency error took too long'
|
||||
elapsed.append(round(result['elapsed'], 3))
|
||||
client.call('GET', '/health/ready', 503)
|
||||
client.call('GET', '/health/live', 200, schema='Health')
|
||||
client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
|
||||
client.call('GET', '/api/v1/users/me', 401, actor={'token': 'unknown'})
|
||||
print(f'PASS {service} unavailable: 503; seconds={elapsed}; anonymous=401; live=200', flush=True)
|
||||
finally:
|
||||
compose('start', service)
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < 60:
|
||||
try:
|
||||
client.call('GET', '/health/ready')
|
||||
break
|
||||
except AssertionError:
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise AssertionError('Storage recovery timed out')
|
||||
client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=actor, schema='MessagePage')['json']
|
||||
assert page['items'] == [message]
|
||||
print(f'PASS {service} recovered in {time.monotonic() - start:.2f}s: same session and message', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from redis.exceptions import ConnectionError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
from scripts.cache_experiment import make_store, stale_race
|
||||
from src.auth import sessions
|
||||
from src.cache import HistoryCache, history_cache
|
||||
from src.database import create_db_engine
|
||||
from src.errors import APIError
|
||||
from src.models import Chat
|
||||
|
||||
|
||||
class CacheTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.engine = create_db_engine()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.engine.dispose()
|
||||
|
||||
def setUp(self):
|
||||
self.client = Client('http://127.0.0.1:8080')
|
||||
self.actor = self.client.user('cache')
|
||||
self.chat = self.client.call('POST', '/api/v1/chats', 201, actor=self.actor,
|
||||
body={'title': 'Cache', 'member_ids': [self.actor['id']]}, schema='Chat')['json']
|
||||
self.prefix = 'test-history:' + uuid4().hex
|
||||
self.cache = HistoryCache(sessions.client, prefix=self.prefix)
|
||||
self.reader = make_store(self.engine, self.cache)
|
||||
self.writer = make_store(self.engine, history_cache)
|
||||
|
||||
def tearDown(self):
|
||||
for key in sessions.client.scan_iter(match=self.prefix + ':*'):
|
||||
sessions.client.delete(key)
|
||||
|
||||
def test_late_fill_cannot_restore_old_page(self):
|
||||
result = stale_race(self.reader, self.writer, self.actor['id'], self.chat['id'])
|
||||
self.assertFalse(result['stale'])
|
||||
|
||||
def test_hit_still_checks_membership_and_cache_failure_falls_back(self):
|
||||
args = (self.chat['id'], self.actor['id'], 100, None)
|
||||
self.writer.create_message(self.chat['id'], self.actor['id'], 'hello')
|
||||
page = self.reader.list_messages(*args)
|
||||
with patch.object(self.reader, '_load_messages', side_effect=AssertionError('Cache miss')):
|
||||
self.assertEqual(self.reader.list_messages(*args), page)
|
||||
with self.assertRaises(APIError) as error:
|
||||
self.reader.list_messages(self.chat['id'], str(uuid4()), 100, None)
|
||||
self.assertEqual(error.exception.status_code, 404)
|
||||
with patch.object(self.cache.client, 'get', side_effect=ConnectionError('unavailable')):
|
||||
self.assertEqual(self.reader.list_messages(*args), page)
|
||||
self.cache.client.delete(self.cache.key(self.chat['id'], 1, 100, None))
|
||||
with patch.object(self.cache.client, 'set', side_effect=ConnectionError('unavailable')):
|
||||
self.assertEqual(self.reader.list_messages(*args), page)
|
||||
|
||||
def test_version_rolls_back_with_failed_message(self):
|
||||
with self.assertRaises(IntegrityError) as error:
|
||||
self.writer.create_message(self.chat['id'], self.actor['id'], '')
|
||||
self.assertEqual(error.exception.orig.sqlstate, '23514')
|
||||
with Session(self.engine) as session:
|
||||
self.assertEqual(session.get(Chat, self.chat['id']).history_version, 0)
|
||||
|
||||
def test_concurrent_writers_do_not_lose_version_increments(self):
|
||||
self.assertEqual(self.reader.list_messages(self.chat['id'], self.actor['id'], 100, None)['items'], [])
|
||||
def write(n):
|
||||
return self.writer.create_message(self.chat['id'], self.actor['id'], f'message {n}')
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
messages = list(executor.map(write, range(8)))
|
||||
with Session(self.engine) as session:
|
||||
self.assertEqual(session.get(Chat, self.chat['id']).history_version, 8)
|
||||
page = self.reader.list_messages(self.chat['id'], self.actor['id'], 100, None)
|
||||
self.assertEqual({item['id'] for item in page['items']}, {item['id'] for item in messages})
|
||||
|
||||
def test_http_read_after_write_and_cursor_pages(self):
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
def read(query=''):
|
||||
return self.client.call('GET', path + query, actor=self.actor, schema='MessagePage')['json']
|
||||
self.assertEqual(read()['items'], [])
|
||||
first = self.client.message(self.actor, self.chat, 'one')['json']
|
||||
self.assertEqual(read()['items'], [first])
|
||||
self.client.message(self.actor, self.chat, 'two')
|
||||
page = read('?limit=1')
|
||||
cursor = quote(page['next_cursor'], safe='')
|
||||
tail = read('?limit=1&cursor=' + cursor)
|
||||
self.assertIsNone(tail['next_cursor'])
|
||||
third = self.client.message(self.actor, self.chat, 'three')['json']
|
||||
tail = read('?limit=1&cursor=' + cursor)
|
||||
self.assertIsNotNone(tail['next_cursor'])
|
||||
self.assertEqual(read('?limit=1&cursor=' + quote(tail['next_cursor'], safe=''))['items'], [third])
|
||||
self.assertEqual(len(read()['items']), 3)
|
||||
with Session(self.engine) as session:
|
||||
version = session.scalar(select(Chat.history_version).where(Chat.id == self.chat['id']))
|
||||
key = history_cache.key(self.chat['id'], version, 50, None)
|
||||
self.assertGreater(sessions.client.ttl(key), 0)
|
||||
self.assertLessEqual(sessions.client.ttl(key), history_cache.ttl)
|
||||
|
||||
def test_read_after_write_through_another_api_process(self):
|
||||
process = subprocess.Popen([sys.executable, '-m', 'src'],
|
||||
env={**os.environ, 'APP_HOST': '127.0.0.1', 'APP_PORT': '8082'},
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
replica = Client('http://127.0.0.1:8082')
|
||||
for _ in range(100):
|
||||
try:
|
||||
replica.call('GET', '/health/ready')
|
||||
break
|
||||
except AssertionError:
|
||||
self.assertIsNone(process.poll())
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
self.fail('Second API did not become ready')
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
self.assertEqual(replica.call('GET', path, actor=self.actor, schema='MessagePage')['json']['items'], [])
|
||||
message = self.client.message(self.actor, self.chat, 'written through first API')['json']
|
||||
self.assertEqual(replica.call('GET', path, actor=self.actor, schema='MessagePage')['json']['items'], [message])
|
||||
finally:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,38 @@
|
||||
import unittest
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import create_engine, inspect, select
|
||||
from sqlalchemy.schema import CreateSchema, DropSchema
|
||||
|
||||
from scripts.cache_experiment import NoCache, make_store
|
||||
from src.database import create_db_engine, migrate
|
||||
from src.models import SchemaMigration
|
||||
|
||||
|
||||
class MigrationTests(unittest.TestCase):
|
||||
def test_migrate_empty_schema_and_preserve_orm_data_on_repeat(self):
|
||||
engine = create_db_engine()
|
||||
schema = 'orm_test_' + uuid4().hex
|
||||
isolated = create_engine(engine.url, connect_args={
|
||||
'connect_timeout': 2, 'options': f'-c search_path={schema} -c statement_timeout=3000'})
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
conn.execute(CreateSchema(schema))
|
||||
migrate(isolated)
|
||||
store = make_store(isolated, NoCache())
|
||||
user = store.create_user('migration_user', 'Migration', 'Synthetic-password-123')
|
||||
chat = store.create_chat(user['id'], 'Migration', [user['id']])
|
||||
message = store.create_message(chat['id'], user['id'], 'must survive migration repeat')
|
||||
migrate(isolated)
|
||||
self.assertEqual(store.list_messages(chat['id'], user['id'], 100, None)['items'], [message])
|
||||
with isolated.connect() as conn:
|
||||
self.assertEqual(set(conn.scalars(select(SchemaMigration.version))),
|
||||
{'001_initial.sql', '002_history_version.sql'})
|
||||
indexes = {index['name'] for index in inspect(isolated).get_indexes('messages')}
|
||||
self.assertIn('messages_history_idx', indexes)
|
||||
self.assertEqual(len(inspect(isolated).get_foreign_keys('messages')), 2)
|
||||
finally:
|
||||
isolated.dispose()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(DropSchema(schema, cascade=True, if_exists=True))
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Run inside the API container: python -m unittest discover -s student-tests -v."""
|
||||
import base64
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
from src.auth import sessions
|
||||
from src.database import create_db_engine, migrate
|
||||
from src.models import User, Chat, ChatMember, SchemaMigration
|
||||
|
||||
|
||||
class Lab2Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.engine = create_db_engine()
|
||||
cls.sessions = sessionmaker(cls.engine)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.engine.dispose()
|
||||
|
||||
def setUp(self):
|
||||
self.client = Client('http://127.0.0.1:8080')
|
||||
|
||||
def test_registration_race(self):
|
||||
body = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Race',
|
||||
'password': 'Synthetic-password-123'}
|
||||
def register(_):
|
||||
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
codes = list(executor.map(register, range(4)))
|
||||
self.assertEqual(sorted(codes), [201, 409, 409, 409])
|
||||
with self.sessions() as session:
|
||||
count = session.scalar(select(func.count()).select_from(User).where(User.username == body['username']))
|
||||
self.assertEqual(count, 1)
|
||||
|
||||
def test_chat_rollback(self):
|
||||
actor = self.client.user('rollback')
|
||||
title = 'rollback_' + uuid4().hex
|
||||
self.client.call('POST', '/api/v1/chats', 400, actor=actor,
|
||||
body={'title': title, 'member_ids': [actor['id'], str(uuid4())]})
|
||||
with self.sessions() as session:
|
||||
self.assertEqual(session.scalar(select(func.count()).select_from(Chat).where(Chat.title == title)), 0)
|
||||
self.assertEqual(session.scalar(select(func.count()).select_from(ChatMember).where(
|
||||
ChatMember.user_id == actor['id'])), 0)
|
||||
|
||||
def test_password_logout_and_identity(self):
|
||||
actor = self.client.user('auth')
|
||||
credential = base64.b64encode((actor['user']['username'] + ':Wrong-password-123').encode()).decode()
|
||||
self.client.call('POST', '/api/v1/auth/sessions', 401,
|
||||
headers={'Authorization': 'Basic ' + credential})
|
||||
self.client.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + credential})
|
||||
self.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
|
||||
me = self.client.call('GET', '/api/v1/users/me', actor=actor,
|
||||
headers={'X-User-Id': str(uuid4())}, schema='User')['json']
|
||||
self.assertEqual(me['id'], actor['id'])
|
||||
ttl = sessions.client.ttl(sessions.key(actor['token']))
|
||||
self.assertGreater(ttl, 0)
|
||||
self.assertLessEqual(ttl, 3600)
|
||||
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
self.client.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=actor)
|
||||
self.assertIsNone(sessions.client.get(sessions.key(actor['token'])))
|
||||
with self.sessions() as session:
|
||||
encoded = session.get(User, actor['id']).password_hash
|
||||
self.assertTrue(encoded.startswith('$argon2id$'))
|
||||
self.assertNotIn(actor['registration']['password'], encoded)
|
||||
|
||||
def test_unicode_password(self):
|
||||
username = 'unicode_' + uuid4().hex[:12]
|
||||
password = '🙂:я' * 40
|
||||
self.client.call('POST', '/api/v1/users', 201,
|
||||
body={'username': username, 'display_name': 'Unicode', 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
result = self.client.call('POST', '/api/v1/auth/sessions', 201,
|
||||
headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
|
||||
self.assertEqual(result['user']['username'], username)
|
||||
|
||||
def test_migrations_repeat(self):
|
||||
with self.sessions() as session:
|
||||
before = session.scalar(select(func.count()).select_from(User))
|
||||
migrate(self.engine)
|
||||
migrate(self.engine)
|
||||
with self.sessions() as session:
|
||||
self.assertEqual(session.scalar(select(func.count()).select_from(User)), before)
|
||||
versions = set(session.scalars(select(SchemaMigration.version)))
|
||||
expected = {path.name for path in (Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')}
|
||||
self.assertEqual(versions, expected)
|
||||
|
||||
def test_expiry_short_ttl(self):
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, '-m', 'src'],
|
||||
env={**os.environ, 'SESSION_TTL_SECONDS': '2', 'APP_PORT': '8081', 'APP_HOST': '127.0.0.1'},
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
client = Client('http://127.0.0.1:8081')
|
||||
for _ in range(100):
|
||||
try:
|
||||
client.call('GET', '/health/ready')
|
||||
break
|
||||
except AssertionError:
|
||||
self.assertIsNone(process.poll(), 'Short-TTL API exited')
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
self.fail('Short-TTL API did not become ready')
|
||||
actor = client.user('expiry')
|
||||
client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
time.sleep(2.2)
|
||||
client.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
self.assertIsNone(sessions.client.get(sessions.key(actor['token'])))
|
||||
finally:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user