Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e84c7a7fc | ||
|
|
175a8d8bfd | ||
|
|
9d3de3b2d9 | ||
|
|
3bd25a9127 |
+16
-4
@@ -3,9 +3,21 @@
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
LOG_LEVEL=info
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# Local synthetic database. Use a password without URL reserved characters.
|
||||
# Compose injects DATABASE_URL into the API from these three values.
|
||||
# The password below is a placeholder: change it in the untracked .env.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace-with-a-local-password
|
||||
POSTGRES_DB=messenger
|
||||
# Optional. When set, the API reads the URL from this file instead of DATABASE_URL.
|
||||
# DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey.
|
||||
# PostgreSQL stores the session. Redis caches the lookup and does not keep its own volume.
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Compose injects REDIS_URL. When set, this file wins and is not logged.
|
||||
# CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# First history page. protected is the default: the page key includes the chat version.
|
||||
# naive and off exist only to measure the track-1 race and must not stay in .env.
|
||||
# HISTORY_CACHE=protected
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# Закреплённая среда выполнения. Тег называет версию, digest — то, что Docker реально скачает.
|
||||
FROM python:3.14.8-slim-bookworm@sha256:c8137f4c460908c8763f281c8f22c431eb5c538514ba9553fc3a89c06b7cfb88
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
# Значения по умолчанию, секретов нет. `docker run --env-file .env` перекрывает их при запуске.
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=5 \
|
||||
SESSION_TTL_SECONDS=3600
|
||||
|
||||
RUN groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install --no-cache-dir --disable-pip-version-check --root-user-action=ignore -r requirements.txt
|
||||
|
||||
COPY --chown=app:app app ./app
|
||||
COPY --chown=app:app migrations ./migrations
|
||||
COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ['APP_PORT'], timeout=2)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -1,37 +1,106 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: Тимаева М.П. БПИ2302
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники, библиотеки и помощь ИИ: контракт и README этой лабораторной; PostgreSQL transactions и EXPLAIN; RFC 7617; библиотеки fastapi, uvicorn, asyncpg 0.32.0, redis 8.1.0, argon2-cffi 25.1.0. Реализацию помог собрать агент Cursor.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker и Docker Compose v2. Образы закреплены: `python:3.14.8-slim-bookworm` (digest в Dockerfile), `postgres:16.10-alpine` и `redis:7.4.11-alpine`.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# В .env замените POSTGRES_PASSWORD на локальный пароль без символов @ : / # ? и пробелов.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
```
|
||||
|
||||
API читает `DATABASE_URL`, который compose собирает из `POSTGRES_USER`, `POSTGRES_PASSWORD` и `POSTGRES_DB`. `REDIS_URL` compose задаёт как `redis://redis:6379/0`. Если заданы `DATABASE_URL_FILE` или `CACHE_URL_FILE`, они важнее и содержимое файлов не логируется. Миграции из `migrations/` применяются при старте API и записываются в `schema_migrations`; повторный старт их не исполняет заново. Пароль БД лежит только в неотслеживаемом `.env`. У Redis пароля нет, своего volume тоже нет.
|
||||
|
||||
Дождитесь `GET http://127.0.0.1:8080/health/ready` → `200 {"status":"ok"}`. Для smoke и persistence `SESSION_TTL_SECONDS` не меньше 300; в `.env.example` стоит 3600.
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test
|
||||
chmod +x scripts/restart.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python scripts/check_grade4.py
|
||||
python scripts/explain_history.py
|
||||
python scripts/track1_cache.py
|
||||
```
|
||||
|
||||
`scripts/restart.sh` перезапускает `api`, `postgres` и `redis`, без `down -v`. На Windows `python` не может запустить этот файл через CreateProcess (`WinError 193`). На этой машине публичный harness был выполнен в контейнере `docker:28-cli` с `python3`, в сети compose и с Docker socket. Подробности в `evidence/grade4.txt`.
|
||||
|
||||
Остановка без удаления данных: `docker compose --env-file .env stop`. Намеренное удаление учебного volume: `docker compose --env-file .env down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент ходит в один процесс API. Источник истины для пользователей, чатов, сообщений и сессий — PostgreSQL, named volume `pgdata`. Redis 7.4 кэширует поиск сессии и первую страницу истории. Ключ сессии — `session:{sha256 токена}`, значение — `expires_at`. Сырой токен и hash пароля в Redis не пишутся. TTL ключа равен остатку до `expires_at` и чтением не продлевается. Промах или недоступный Redis не заменяют строку PostgreSQL: личность подтверждается только ею. Logout сначала удаляет строку, затем ключ.
|
||||
|
||||
Первая страница `GET /messages` без cursor при `limit=50` лежит в ключе `history:{chat_id}:{history_version}`. Колонка `chats.history_version` увеличивается в той же транзакции, что и `INSERT` сообщения, и публикуется в Redis до ответа 201. Чтение, которое начинается после 201, видит новую версию и не читает страницу предыдущей. Гонка «SELECT успел выполниться до commit и записал старую страницу» публикует её только под старой версией. TTL страницы 30 с снимает брошенные ключи и не отвечает за свежесть. Страницы с cursor всегда идут в SQL. Пока один промах выполняет SELECT, остальные запросы того же процесса ждут тот же результат (singleflight на asyncio future). `HISTORY_CACHE=naive` пишет страницу в ключ без версии и не сбрасывает её при записи; `off` не кэширует историю. Оба режима только для измерения, обычный `docker compose up` оставляет `protected`.
|
||||
|
||||
Redis запущен как `redis-server --save "" --appendonly no` и без своего volume. Рестарт контейнера поднимает пустой кэш. После этого сессия читается из PostgreSQL, а история — тоже из PostgreSQL, пока страница снова не попадёт в кэш.
|
||||
|
||||
Таблицы `migrations/001_init.sql`: `users` (UNIQUE username), `sessions` (PK — SHA-256 токена, FK на пользователя, `expires_at`), `chats`, `chat_members` (PK пары, оба FK, `position`), `messages` (FK на чат и отправителя), `schema_migrations`. `migrations/002_chat_version.sql` добавляет `chats.history_version bigint NOT NULL DEFAULT 0`. Индекс `messages_history_idx (chat_id, created_at, id)` обслуживает keyset. Запросы параметризованы (`$1` в asyncpg). Пул: `min_size` 1, `max_size` 10, таймаут подключения и команды 5 с, `statement_timeout` 5000 мс. Парольный hash в ответы не выбирается. Клиенту отдаётся сырой opaque-токен (`secrets.token_urlsafe(32)`).
|
||||
|
||||
Создание чата — одна транзакция: все `member_ids` должны существовать, иначе rollback и 400, затем вставка чата и участников. Занятый username — unique violation → 409 без текста SQL. `created_at` назначает процесс и делает его строго возрастающим, чтобы порядок `(created_at, id)` совпадал с порядком записи. Рестарт не меняет `expires_at`.
|
||||
|
||||
`GET /health/live` проверяет только процесс. `GET /health/ready` делает `SELECT 1`; без БД ответ 503 `unavailable`. Та же 503, а не 500 и не личность из кэша, приходит на Bearer, если PostgreSQL недоступен. Запрос без токена остаётся 401 и до Redis, и до БД. Basic принимается только на `POST /api/v1/auth/sessions`. Остальные маршруты требуют Bearer. `X-User-Id` не идентифицирует пользователя. Пароли — Argon2id.
|
||||
|
||||
Переменные: `APP_HOST`, `APP_PORT`, `APP_ENV`, `LOG_LEVEL`, `SHUTDOWN_TIMEOUT_SECONDS`, `SESSION_TTL_SECONDS` читаются как в `.env.example`. `DATABASE_URL` и `REDIS_URL` задаёт compose; `DATABASE_URL_FILE` и `CACHE_URL_FILE` — необязательные замены. `SESSION_STORE=postgres` означает, что строка сессии живёт в PostgreSQL, а Redis только кэширует поиск. `HISTORY_CACHE` по умолчанию `protected`.
|
||||
|
||||
Из lab1 сохранены пользователи, чаты, членство, сообщения и keyset-пагинация. Идентификация lab1 (`X-User-Id`) заменена на пароль, Basic и Bearer.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` на образе из Dockerfile | PASS: lab2 public API scenario | evidence/grade3.txt |
|
||||
| Данные и сессия переживают рестарт API, PostgreSQL и Redis | `tests/resilience.py --mode persistence --action-script scripts/restart.sh` | PASS. ID не сменились (`87a78974fbc8`, `cbe7139645a5`, `e271b1f6ea0e`), volume `mtusi2026lab2-fork_pgdata` на месте, у Redis volume нет | evidence/grade4.txt |
|
||||
| Гонка username, rollback, неверный пароль, logout, короткий TTL, отказ Redis и PostgreSQL | `python scripts/check_grade4.py` | Все семь проверок PASS. После скрипта TTL снова 3600 | evidence/grade4.txt |
|
||||
| Индекс истории | `python scripts/explain_history.py`, 10 000 сообщений в одном чате | `Index Scan using messages_history_idx`, LIMIT 51, `Buffers: shared hit=4`, Seq Scan нет | evidence/explain.txt |
|
||||
| Кэш первой страницы истории | `python scripts/track1_cache.py`, один чат, `limit=50`, без другой нагрузки на `messages` | naive: 8 из 8 чтений после 201 без нового сообщения; 32 холодных GET → 32 скана. off: 20 GET → 20 сканов. protected: 0 из 8 пропусков; 32 GET → 1 скан; 20 тёплых → 0 сканов, p50 5.786 мс против 8.519 мс у холодного промаха. Рестарт Redis: сообщение на месте, сканы 1 затем 0, сессия 200 | evidence/track1.json |
|
||||
| Повтор миграций не дублирует данные | второй `docker compose restart api`, затем counts | `schema_migrations` = 1, users = 14, messages = 6 и до, и после | evidence/grade3.txt |
|
||||
| Атомарное создание чата | `python scripts/check_grade3.py` | 400 на неизвестного участника, в `chats` строки с этим title нет | evidence/grade3.txt |
|
||||
| Expiry сессии | отдельный прогон с `SESSION_TTL_SECONDS=2`, затем возврат к 3600 | тот же токен через 3 с получил 401 | evidence/grade3.txt |
|
||||
| Недоступная БД | `docker compose stop postgres`, затем start | `/health/live` 200, `/health/ready` 503 `unavailable` без SQL | evidence/grade3.txt |
|
||||
| Секреты вне Git | `git check-ignore -v .env` | `.gitignore:2:.env` | — |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
`make check` эквивалентен `python scripts/check_contract.py` и тоже PASS. Контракт и публичные тесты не менялись. После трека 1 заново прогнаны smoke (PASS) и persistence (PASS). ID после этого рестарта: api `1538b7d21085`, postgres `cbe7139645a5`, redis `c6a09b1c7977`. Volume `mtusi2026lab2-fork_pgdata` на месте. `HISTORY_CACHE` в контейнере `protected`, в `.env` этой переменной нет, `SESSION_TTL_SECONDS` 3600.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза трека 1: без версии чата чтение, начатое после уже подтверждённой записи, может получить первую страницу без этого сообщения; версия в ключе убирает такое чтение; singleflight на холодном ключе сводит SELECT истории к одному; тёплая страница не трогает `messages` и короче холодного промаха.
|
||||
|
||||
Методика. Один чат, один и тот же `GET /messages?limit=50`, стенд без другой нагрузки на `messages`. Счётчик — дельта `idx_scan + seq_scan` из `pg_stat_user_tables` для `messages`. PostgreSQL публикует эти числа при завершении backend, поэтому скрипт после каждого окна завершает соединения пула и только потом читает статистику. p50 считается по успешным GET на клиенте. В `naive` процесс держит заполнение страницы 200 мс, чтобы 32 промаха успели открыть свой SELECT; в `protected` и `off` этой паузы нет. Скрипт сам переключает `HISTORY_CACHE`, пересоздаёт только API и в `finally` возвращает прежний `.env`.
|
||||
|
||||
| Режим | Наблюдение | Сканы `messages` | p50, мс |
|
||||
| --- | --- | --- | --- |
|
||||
| naive, 8 GET, каждый начат после 201 | все 8 без нового сообщения | — | — |
|
||||
| naive, 32 одновременных GET по холодному ключу | отдельный SELECT на клиента | 32 | 343.766, внутри пауза записи 200 мс |
|
||||
| off, 20 последовательных GET | кэша нет, каждый GET читает SQL | 20 | 4.950 |
|
||||
| protected, 8 GET, каждый начат после 201 | новое сообщение есть в каждом | — | — |
|
||||
| protected, 32 одновременных GET по холодному ключу | один SELECT на всех | 1 | 160.808 |
|
||||
| protected, холодный промах по одному | страница строится заново | — | 8.519 |
|
||||
| protected, 20 GET по тёплой странице | таблица `messages` не читается | 0 | 5.786 |
|
||||
| `docker compose restart redis`, PostgreSQL жив | сообщение на месте; первое чтение — промах, второе — попадание; `GET /users/me` → 200 | 1, затем 0 | — |
|
||||
|
||||
Тёплое попадание (5.786 мс) короче холодного промаха (8.519 мс). Один короткий index scan в режиме off дал 4.950 мс: на этом стенде два обращения к Redis близки по времени к одному такому SELECT. Выигрыш тёплой страницы здесь — ноль сканов `messages`.
|
||||
|
||||
EXPLAIN снят на 10 000 сообщениях одного чата: вторая страница keyset, `LIMIT 51`, выполнение 0.363 мс, прочитаны 4 страницы буфера.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `scripts/restart.sh` перезапускает api, postgres и redis | Тот же Bearer читает прежнее сообщение, volume PostgreSQL сохранён, кэш может опустеть | PASS persistence; ID трёх контейнеров те же, volume на месте | Готовность уложилась в бюджет теста (90 с). Подтверждённые строки не пропали |
|
||||
| Остановлен только Redis | Действующая сессия читается из PostgreSQL, аноним получает 401 | Так и было | После `start` кэш снова доступен. Данные не терялись |
|
||||
| Остановлен только postgres | Bearer получает 503 без SQL, аноним — 401 | Так и было | После `start` ready снова 200. Volume не удалялся |
|
||||
| `docker compose restart redis` при живом PostgreSQL | Подтверждённое сообщение остаётся, первое чтение истории идёт в SQL, следующее берётся из кэша, сессия жива | Сообщение на месте, сканы `messages` 1 затем 0, сессия 200 | Кэш после рестарта пуст, потому что RDB и AOF выключены и volume нет. Строки PostgreSQL не пропадали |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Один процесс API и одна БД. Потеря Docker-хоста уничтожает и API, и PostgreSQL. Redis не хранит сессию сам по себе: после очистки кэша доступна только строка в PostgreSQL, а пока БД недоступна, кэш не принимается за личность. Пул ограничен 10 соединениями и таймаутом 5 с. Идемпотентность и вторая реплика — лабораторная 3 курса.
|
||||
|
||||
Singleflight действует внутри одного процесса. Страницы с cursor не кэшируются. Дельта `pg_stat_user_tables` верна, пока во время окна нет другой нагрузки на `messages`. TTL 30 с не заменяет версию в ключе. Пауза 200 мс есть только в `naive`, чтобы одновременные промахи наложились; она входит в p50 этого режима. На этом стенде один index scan короче двух чтений Redis, поэтому тёплый кэш выигрывает числом SQL, а не каждым миллисекундом относительно `off`.
|
||||
|
||||
Учебное упрощение: время сообщения монотонно только внутри одного процесса. Процесс один, поэтому порядок записи и `(created_at, id)` совпадают.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import uvicorn
|
||||
|
||||
from .config import load_settings
|
||||
|
||||
|
||||
def main() -> None:
|
||||
settings = load_settings()
|
||||
uvicorn.run(
|
||||
'app.main:app',
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
log_level=settings.log_level,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Basic is only for creating a session. Every other route accepts only Bearer."""
|
||||
import base64
|
||||
import binascii
|
||||
from typing import Optional, Tuple
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def parse_basic(header: str) -> Optional[Tuple[str, str]]:
|
||||
"""username:password. The password may itself contain a colon."""
|
||||
scheme, separator, param = header.partition(' ')
|
||||
if separator != ' ' or scheme.lower() != 'basic' or not param or param != param.strip():
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(param, validate=True)
|
||||
text = decoded.decode('utf-8')
|
||||
except (binascii.Error, UnicodeDecodeError, ValueError):
|
||||
return None
|
||||
username, separator, password = text.partition(':')
|
||||
if separator != ':':
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: str) -> Optional[str]:
|
||||
scheme, separator, token = header.partition(' ')
|
||||
if separator != ' ' or scheme.lower() != 'bearer' or not token or token != token.strip() or ' ' in token:
|
||||
return None
|
||||
return token
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Lookup cache for sessions and the first history page. PostgreSQL remains the source of truth."""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_PREFIX = 'session:'
|
||||
_VERSION_PREFIX = 'hver:'
|
||||
_PAGE_TTL_SECONDS = 30
|
||||
|
||||
|
||||
def open_cache(url: str) -> Redis:
|
||||
return Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=1,
|
||||
max_connections=10,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
def _stamp(moment: datetime) -> str:
|
||||
return moment.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
class SessionCache:
|
||||
"""Stores sha256(token) -> expires_at. A read never refreshes the TTL."""
|
||||
|
||||
def __init__(self, client: Redis) -> None:
|
||||
self._client = client
|
||||
|
||||
async def get(self, token_hash: str) -> Optional[str]:
|
||||
"""Cached expiry, or None on a miss and when Redis cannot be reached."""
|
||||
try:
|
||||
return await self._client.get(_PREFIX + token_hash)
|
||||
except Exception as exc:
|
||||
logger.warning('session cache read skipped (%s)', type(exc).__name__)
|
||||
return None
|
||||
|
||||
async def remember(
|
||||
self,
|
||||
token_hash: str,
|
||||
expires_at: datetime,
|
||||
now: datetime,
|
||||
*,
|
||||
only_if_absent: bool,
|
||||
) -> None:
|
||||
remaining_ms = int((expires_at - now).total_seconds() * 1000)
|
||||
if remaining_ms <= 0:
|
||||
await self.drop(token_hash)
|
||||
return
|
||||
try:
|
||||
await self._client.set(
|
||||
_PREFIX + token_hash,
|
||||
_stamp(expires_at),
|
||||
px=remaining_ms,
|
||||
nx=only_if_absent,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning('session cache write skipped (%s)', type(exc).__name__)
|
||||
|
||||
async def drop(self, token_hash: str) -> None:
|
||||
try:
|
||||
await self._client.delete(_PREFIX + token_hash)
|
||||
except Exception as exc:
|
||||
logger.warning('session cache delete skipped (%s)', type(exc).__name__)
|
||||
|
||||
async def history_version(self, chat_id: str) -> Optional[int]:
|
||||
"""Published chat version, or None on a miss and when Redis cannot be reached."""
|
||||
try:
|
||||
raw = await self._client.get(_VERSION_PREFIX + chat_id)
|
||||
except Exception as exc:
|
||||
logger.warning('history version read skipped (%s)', type(exc).__name__)
|
||||
return None
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def publish_history_version(self, chat_id: str, version: int) -> None:
|
||||
"""Point readers at version. A failed write drops the key so the next read uses PostgreSQL."""
|
||||
key = _VERSION_PREFIX + chat_id
|
||||
try:
|
||||
await self._client.set(key, str(version))
|
||||
except Exception as exc:
|
||||
logger.warning('history version write skipped (%s)', type(exc).__name__)
|
||||
try:
|
||||
await self._client.delete(key)
|
||||
except Exception as drop_exc:
|
||||
logger.warning('history version delete skipped (%s)', type(drop_exc).__name__)
|
||||
|
||||
async def get_history_page(self, key: str) -> Optional[str]:
|
||||
try:
|
||||
return await self._client.get(key)
|
||||
except Exception as exc:
|
||||
logger.warning('history cache read skipped (%s)', type(exc).__name__)
|
||||
return None
|
||||
|
||||
async def set_history_page(self, key: str, payload: str) -> None:
|
||||
"""TTL only drops abandoned version keys. Freshness comes from the version in the key."""
|
||||
try:
|
||||
await self._client.set(key, payload, ex=_PAGE_TTL_SECONDS)
|
||||
except Exception as exc:
|
||||
logger.warning('history cache write skipped (%s)', type(exc).__name__)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
host: str
|
||||
port: int
|
||||
env: str
|
||||
log_level: str
|
||||
shutdown_timeout: int
|
||||
session_ttl_seconds: int
|
||||
database_url: str
|
||||
redis_url: str
|
||||
history_cache: str
|
||||
|
||||
|
||||
def _secret(file_var: str, value_var: str) -> str:
|
||||
"""A *_FILE variable wins. The secret value is not logged."""
|
||||
file_name = os.getenv(file_var, '').strip()
|
||||
if file_name:
|
||||
try:
|
||||
value = Path(file_name).read_text(encoding='utf-8').strip()
|
||||
except OSError:
|
||||
raise RuntimeError('%s could not be read' % file_var) from None
|
||||
if not value:
|
||||
raise RuntimeError('%s is empty' % file_var)
|
||||
return value
|
||||
value = os.getenv(value_var, '').strip()
|
||||
if not value:
|
||||
raise RuntimeError('%s or %s is required' % (value_var, file_var))
|
||||
return value
|
||||
|
||||
|
||||
def _history_cache() -> str:
|
||||
mode = os.getenv('HISTORY_CACHE', 'protected').strip().lower() or 'protected'
|
||||
if mode not in ('protected', 'naive', 'off'):
|
||||
raise RuntimeError('HISTORY_CACHE must be protected, naive or off')
|
||||
return mode
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||
port=int(os.getenv('APP_PORT', '8080')),
|
||||
env=os.getenv('APP_ENV', 'lab'),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info').lower(),
|
||||
shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '5')),
|
||||
session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')),
|
||||
database_url=_secret('DATABASE_URL_FILE', 'DATABASE_URL'),
|
||||
redis_url=_secret('CACHE_URL_FILE', 'REDIS_URL'),
|
||||
history_cache=_history_cache(),
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Connection pool. The URL is never written to the log."""
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import asyncpg
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_ATTEMPTS = 40
|
||||
|
||||
|
||||
async def open_pool(database_url: str) -> asyncpg.Pool:
|
||||
delay = 0.5
|
||||
last_type = 'error'
|
||||
for attempt in range(1, _ATTEMPTS + 1):
|
||||
pool = None
|
||||
try:
|
||||
pool = await asyncpg.create_pool(
|
||||
database_url,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
timeout=5,
|
||||
command_timeout=5,
|
||||
server_settings={'statement_timeout': '5000'},
|
||||
)
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute('SELECT 1')
|
||||
return pool
|
||||
except Exception as exc:
|
||||
last_type = type(exc).__name__
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
if attempt == _ATTEMPTS:
|
||||
break
|
||||
logger.warning('database is not ready (%s), retrying', last_type)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 1.5, 2)
|
||||
raise RuntimeError('database did not become reachable (%s)' % last_type)
|
||||
@@ -0,0 +1,29 @@
|
||||
class ApiError(Exception):
|
||||
"""Expected failure: the message never contains internal details."""
|
||||
|
||||
def __init__(self, status: int, code: str, message: str, challenge: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.challenge = challenge
|
||||
|
||||
|
||||
def invalid_request(message: str) -> ApiError:
|
||||
return ApiError(400, 'invalid_request', message)
|
||||
|
||||
|
||||
def unauthorized(message: str, challenge: str) -> ApiError:
|
||||
return ApiError(401, 'unauthorized', message, challenge)
|
||||
|
||||
|
||||
def not_found(message: str) -> ApiError:
|
||||
return ApiError(404, 'not_found', message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, 'conflict', message)
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, 'unavailable', message)
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from .cache import SessionCache, open_cache
|
||||
from .config import load_settings
|
||||
from .db import open_pool
|
||||
from .errors import ApiError
|
||||
from .migrate import apply_migrations
|
||||
from .routes import router
|
||||
from .store import Store
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_SAFE_REQUEST_ID = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
|
||||
_CODES = {
|
||||
400: 'invalid_request',
|
||||
401: 'unauthorized',
|
||||
404: 'not_found',
|
||||
409: 'conflict',
|
||||
413: 'payload_too_large',
|
||||
415: 'unsupported_media_type',
|
||||
429: 'rate_limited',
|
||||
503: 'unavailable',
|
||||
}
|
||||
_MESSAGES = {
|
||||
'invalid_request': 'Некорректный запрос',
|
||||
'unauthorized': 'Требуется идентификация',
|
||||
'not_found': 'Объект не найден',
|
||||
'internal_error': 'Внутренняя ошибка',
|
||||
}
|
||||
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def _error(status: int, code: str, message: str, request_id: str, challenge: str | None = None) -> JSONResponse:
|
||||
body = {'error': {'code': code, 'message': message, 'request_id': request_id}}
|
||||
headers = {'X-Request-Id': request_id}
|
||||
if challenge:
|
||||
headers['WWW-Authenticate'] = challenge
|
||||
return JSONResponse(body, status_code=status, headers=headers)
|
||||
|
||||
|
||||
def _request_id(request: Request) -> str:
|
||||
return getattr(request.state, 'request_id', None) or str(uuid.uuid4())
|
||||
|
||||
|
||||
_OPENAPI_PATH = Path(__file__).resolve().parents[1] / 'contracts' / 'openapi.json'
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
settings = load_settings()
|
||||
pool = await open_pool(settings.database_url)
|
||||
cache = SessionCache(open_cache(settings.redis_url))
|
||||
try:
|
||||
await apply_migrations(pool)
|
||||
app.state.store = Store(
|
||||
pool,
|
||||
cache,
|
||||
settings.session_ttl_seconds,
|
||||
settings.history_cache,
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
await cache.close()
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(
|
||||
title='MTUSI Messenger lab2',
|
||||
version='1.2.0',
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
def openapi():
|
||||
if app.openapi_schema is None:
|
||||
app.openapi_schema = json.loads(_OPENAPI_PATH.read_text(encoding='utf-8'))
|
||||
return app.openapi_schema
|
||||
|
||||
app.openapi = openapi
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
"""Gives every response, including unexpected failures, an X-Request-Id."""
|
||||
incoming = request.headers.get('X-Request-Id')
|
||||
request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
logger.exception('request %s failed: %s %s', request_id, request.method, request.url.path)
|
||||
response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id)
|
||||
response.headers['X-Request-Id'] = request_id
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse:
|
||||
return _error(exc.status, exc.code, exc.message, _request_id(request), exc.challenge)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return _error(400, 'invalid_request', _MESSAGES['invalid_request'], _request_id(request))
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||
code = _CODES.get(exc.status_code, 'internal_error' if exc.status_code >= 500 else 'invalid_request')
|
||||
challenge = _BEARER_CHALLENGE if exc.status_code == 401 else None
|
||||
return _error(exc.status_code, code, _MESSAGES.get(code, _MESSAGES['invalid_request']), _request_id(request), challenge)
|
||||
|
||||
app.include_router(router)
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Apply numbered SQL files once. Re-running records nothing new and inserts no rows."""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_MIGRATIONS = Path(__file__).resolve().parents[1] / 'migrations'
|
||||
|
||||
_BOOTSTRAP = """
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
async def apply_migrations(pool: asyncpg.Pool) -> None:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(_BOOTSTRAP)
|
||||
rows = await conn.fetch('SELECT version FROM schema_migrations')
|
||||
applied = {row['version'] for row in rows}
|
||||
for path in sorted(_MIGRATIONS.glob('*.sql')):
|
||||
version = path.name
|
||||
if version in applied:
|
||||
logger.info('migration %s already applied', version)
|
||||
continue
|
||||
sql = path.read_text(encoding='utf-8')
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute(
|
||||
'INSERT INTO schema_migrations (version) VALUES ($1)',
|
||||
version,
|
||||
)
|
||||
logger.info('applied migration %s', version)
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from typing import Mapping, Optional, Tuple
|
||||
|
||||
from .errors import invalid_request
|
||||
|
||||
# Position of the last item returned to the client.
|
||||
Key = Tuple[str, str]
|
||||
|
||||
DEFAULT_LIMIT = 50
|
||||
MAX_LIMIT = 100
|
||||
_LIMIT = re.compile(r'^\d{1,3}$')
|
||||
|
||||
|
||||
def encode_cursor(scope: str, key: Key) -> str:
|
||||
"""Scope binds the cursor to one resource, so a foreign cursor is detectable."""
|
||||
payload = json.dumps({'s': scope, 't': key[0], 'i': key[1]}, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(payload.encode('utf-8')).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(value: str, scope: str) -> Key:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
|
||||
payload = json.loads(raw)
|
||||
created_at, item_id = payload['t'], payload['i']
|
||||
belongs = payload['s'] == scope
|
||||
except (ValueError, TypeError, KeyError):
|
||||
raise invalid_request('Некорректный cursor') from None
|
||||
if not belongs or not isinstance(created_at, str) or not isinstance(item_id, str):
|
||||
raise invalid_request('Cursor относится к другому запросу') from None
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
def parse_page(params: Mapping[str, str], scope: str) -> Tuple[int, Optional[Key]]:
|
||||
raw_limit = params.get('limit')
|
||||
limit = DEFAULT_LIMIT
|
||||
if raw_limit is not None:
|
||||
if not _LIMIT.match(raw_limit) or not 1 <= int(raw_limit) <= MAX_LIMIT:
|
||||
raise invalid_request('limit должен быть целым числом от 1 до %d' % MAX_LIMIT)
|
||||
limit = int(raw_limit)
|
||||
raw_cursor = params.get('cursor')
|
||||
return limit, decode_cursor(raw_cursor, scope) if raw_cursor is not None else None
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Argon2id password hashes. The hash never leaves this module in an API response."""
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
# Same cost as a real hash, so a missing user is not faster than a wrong password.
|
||||
_DUMMY_HASH = _hasher.hash('not-a-user-password')
|
||||
_DUMMY_PASSWORD = 'not-a-user-password'
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except VerificationError:
|
||||
return False
|
||||
|
||||
|
||||
def password_matches(stored_hash: str | None, password: str) -> bool:
|
||||
"""One Argon2 check either way. An out-of-range secret is never the stored password."""
|
||||
if stored_hash is None or not 12 <= len(password) <= 128 or not password.strip():
|
||||
verify_password(_DUMMY_HASH, _DUMMY_PASSWORD)
|
||||
return False
|
||||
return verify_password(stored_hash, password)
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
import json
|
||||
import uuid
|
||||
from typing import Optional, Sequence, Type, TypeVar
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from starlette.responses import Response
|
||||
|
||||
from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer
|
||||
from .errors import invalid_request, not_found, unauthorized
|
||||
from .pagination import encode_cursor, parse_page
|
||||
from .schemas import CreateChat, CreateMessage, CreateUser
|
||||
from .store import Chat, Store, User
|
||||
|
||||
router = APIRouter()
|
||||
Body = TypeVar('Body', bound=BaseModel)
|
||||
|
||||
|
||||
async def get_store(request: Request) -> Store:
|
||||
return request.app.state.store
|
||||
|
||||
|
||||
async def current_user(request: Request) -> User:
|
||||
"""Bearer only. X-User-Id is ignored and does not identify anyone."""
|
||||
token = parse_bearer(request.headers.get('Authorization', ''))
|
||||
if token is None:
|
||||
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
|
||||
user = await request.app.state.store.user_for_token(token)
|
||||
if user is None:
|
||||
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
|
||||
return user
|
||||
|
||||
|
||||
async def read_body(request: Request, model: Type[Body]) -> Body:
|
||||
try:
|
||||
payload = json.loads(await request.body())
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise invalid_request('Тело запроса не является корректным JSON') from None
|
||||
if not isinstance(payload, dict):
|
||||
raise invalid_request('Тело запроса должно быть JSON-объектом')
|
||||
try:
|
||||
return model.model_validate(payload)
|
||||
except ValidationError:
|
||||
raise invalid_request('Тело запроса не соответствует контракту') from None
|
||||
|
||||
|
||||
async def _member_chat(store: Store, chat_id: str, actor: User) -> Chat:
|
||||
try:
|
||||
canonical = str(uuid.UUID(chat_id))
|
||||
except ValueError:
|
||||
raise invalid_request('Некорректный chat_id') from None
|
||||
chat = await store.find_chat(canonical, actor.id)
|
||||
if chat is None:
|
||||
raise not_found('Чат не найден')
|
||||
return chat
|
||||
|
||||
|
||||
def _page(items: Sequence, has_more: bool, scope: str) -> dict:
|
||||
cursor: Optional[str] = None
|
||||
if has_more and items:
|
||||
cursor = encode_cursor(scope, (items[-1].created_at, items[-1].id))
|
||||
return {'items': [item.to_dict() for item in items], 'next_cursor': cursor}
|
||||
|
||||
|
||||
@router.get('/health/live')
|
||||
async def liveness() -> JSONResponse:
|
||||
return JSONResponse({'status': 'ok'})
|
||||
|
||||
|
||||
@router.get('/health/ready')
|
||||
async def readiness(store: Store = Depends(get_store)) -> JSONResponse:
|
||||
await store.ready()
|
||||
return JSONResponse({'status': 'ok'})
|
||||
|
||||
|
||||
@router.post('/api/v1/users')
|
||||
async def register_user(request: Request, store: Store = Depends(get_store)) -> JSONResponse:
|
||||
payload = await read_body(request, CreateUser)
|
||||
user = await store.create_user(payload.username, payload.display_name, payload.password)
|
||||
return JSONResponse(user.to_dict(), status_code=201)
|
||||
|
||||
|
||||
@router.post('/api/v1/auth/sessions')
|
||||
async def create_session(request: Request, store: Store = Depends(get_store)) -> JSONResponse:
|
||||
parsed = parse_basic(request.headers.get('Authorization', ''))
|
||||
if parsed is None:
|
||||
raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE)
|
||||
user = await store.authenticate(*parsed)
|
||||
if user is None:
|
||||
raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE)
|
||||
token, expires_at = await store.create_session(user.id)
|
||||
body = {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': expires_at,
|
||||
'user': user.to_dict(),
|
||||
}
|
||||
return JSONResponse(body, status_code=201)
|
||||
|
||||
|
||||
@router.delete('/api/v1/auth/sessions/current')
|
||||
async def revoke_session(request: Request, actor: User = Depends(current_user)) -> Response:
|
||||
token = parse_bearer(request.headers.get('Authorization', ''))
|
||||
if token is None or not await request.app.state.store.revoke_session(token):
|
||||
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.get('/api/v1/users/me')
|
||||
async def show_current_user(actor: User = Depends(current_user)) -> JSONResponse:
|
||||
return JSONResponse(actor.to_dict())
|
||||
|
||||
|
||||
@router.post('/api/v1/chats')
|
||||
async def create_chat(
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
payload = await read_body(request, CreateChat)
|
||||
if actor.id not in payload.member_ids:
|
||||
raise invalid_request('Создатель чата должен быть указан в member_ids')
|
||||
chat = await store.create_chat(payload.title, payload.member_ids)
|
||||
return JSONResponse(chat.to_dict(), status_code=201)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats')
|
||||
async def list_chats(
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
scope = 'chats:' + actor.id
|
||||
limit, after = parse_page(request.query_params, scope)
|
||||
chats, has_more = await store.list_chats(actor.id, limit, after)
|
||||
return JSONResponse(_page(chats, has_more, scope))
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}')
|
||||
async def show_chat(
|
||||
chat_id: str,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
return JSONResponse((await _member_chat(store, chat_id, actor)).to_dict())
|
||||
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/messages')
|
||||
async def send_message(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
chat = await _member_chat(store, chat_id, actor)
|
||||
payload = await read_body(request, CreateMessage)
|
||||
message = await store.add_message(chat.id, actor.id, payload.text)
|
||||
return JSONResponse(message.to_dict(), status_code=201)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}/messages')
|
||||
async def list_messages(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
chat = await _member_chat(store, chat_id, actor)
|
||||
scope = 'messages:' + chat.id
|
||||
limit, after = parse_page(request.query_params, scope)
|
||||
messages, has_more = await store.list_messages(chat.id, limit, after)
|
||||
return JSONResponse(_page(messages, has_more, scope))
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Request bodies. The contract forbids unknown fields, wrong types and blank strings."""
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator
|
||||
from typing_extensions import Annotated
|
||||
|
||||
|
||||
def _not_blank(value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError('value must not be whitespace only')
|
||||
return value
|
||||
|
||||
|
||||
def _canonical_uuid(value: str) -> str:
|
||||
parsed = uuid.UUID(value)
|
||||
if str(parsed) != value.lower():
|
||||
raise ValueError('value must be a canonical UUID')
|
||||
return str(parsed)
|
||||
|
||||
|
||||
Username = Annotated[str, Field(pattern=r'^[a-z0-9_]{3,32}$')]
|
||||
DisplayName = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)]
|
||||
Title = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)]
|
||||
Text = Annotated[str, Field(min_length=1, max_length=4000), AfterValidator(_not_blank)]
|
||||
Password = Annotated[str, Field(min_length=12, max_length=128), AfterValidator(_not_blank)]
|
||||
UserId = Annotated[str, AfterValidator(_canonical_uuid)]
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
username: Username
|
||||
display_name: DisplayName
|
||||
password: Password
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
title: Title
|
||||
member_ids: Annotated[List[UserId], Field(min_length=1, max_length=100)]
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def _unique(cls, value: List[str]) -> List[str]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('member_ids must be unique')
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
text: Text
|
||||
+537
@@ -0,0 +1,537 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import asyncpg
|
||||
|
||||
from .cache import SessionCache
|
||||
from .errors import conflict, invalid_request, not_found, unavailable
|
||||
from .pagination import Key
|
||||
from .passwords import hash_password, password_matches
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'username': self.username,
|
||||
'display_name': self.display_name,
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: Tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'member_ids': list(self.member_ids),
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'chat_id': self.chat_id,
|
||||
'sender_id': self.sender_id,
|
||||
'text': self.text,
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
|
||||
|
||||
def _token_key(token: str) -> str:
|
||||
return hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def _stamp(moment: datetime) -> str:
|
||||
return moment.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def _encode_page(messages: Sequence[Message], has_more: bool) -> str:
|
||||
return json.dumps(
|
||||
{'items': [message.to_dict() for message in messages], 'has_more': has_more},
|
||||
separators=(',', ':'),
|
||||
)
|
||||
|
||||
|
||||
def _decode_page(payload: Optional[str]) -> Optional[Tuple[List[Message], bool]]:
|
||||
if not payload:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
messages = [
|
||||
Message(item['id'], item['chat_id'], item['sender_id'], item['text'], item['created_at'])
|
||||
for item in data['items']
|
||||
]
|
||||
return messages, bool(data['has_more'])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_stamp(value: str) -> datetime:
|
||||
if value.endswith('Z'):
|
||||
value = value[:-1] + '+00:00'
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def _user(row: asyncpg.Record) -> User:
|
||||
return User(str(row['id']), row['username'], row['display_name'], _stamp(row['created_at']))
|
||||
|
||||
|
||||
def _as_uuid(value: str, message: str) -> uuid.UUID:
|
||||
try:
|
||||
return uuid.UUID(value)
|
||||
except ValueError:
|
||||
raise invalid_request(message) from None
|
||||
|
||||
|
||||
def _db_down(exc: BaseException) -> bool:
|
||||
return isinstance(exc, (
|
||||
asyncpg.PostgresConnectionError,
|
||||
asyncpg.InterfaceError,
|
||||
asyncpg.CannotConnectNowError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
))
|
||||
|
||||
|
||||
_FIRST_PAGE_LIMIT = 50
|
||||
# Opens the unguarded fill long enough for concurrent misses to each run their own SELECT.
|
||||
_NAIVE_FILL_HOLD_SECONDS = 0.2
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL is the source of truth. Redis caches session lookup and the first history page."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pool: asyncpg.Pool,
|
||||
cache: SessionCache,
|
||||
session_ttl_seconds: int = 3600,
|
||||
history_mode: str = 'protected',
|
||||
) -> None:
|
||||
self._pool = pool
|
||||
self._cache = cache
|
||||
self._session_ttl = timedelta(seconds=session_ttl_seconds)
|
||||
self._history_mode = history_mode
|
||||
self._lock = threading.Lock()
|
||||
self._last_moment: Optional[datetime] = None
|
||||
self._flights: Dict[str, asyncio.Future] = {}
|
||||
self._flight_lock = asyncio.Lock()
|
||||
|
||||
def _now(self) -> datetime:
|
||||
"""Strictly increasing server time, so append order always equals sort order."""
|
||||
with self._lock:
|
||||
moment = datetime.now(timezone.utc)
|
||||
if self._last_moment is not None and moment <= self._last_moment:
|
||||
moment = self._last_moment + timedelta(microseconds=1)
|
||||
self._last_moment = moment
|
||||
return moment
|
||||
|
||||
async def ready(self) -> None:
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute('SELECT 1')
|
||||
except Exception:
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
|
||||
async def create_user(self, username: str, display_name: str, password: str) -> User:
|
||||
password_hash = hash_password(password)
|
||||
user_id = uuid.uuid4()
|
||||
created = self._now()
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
''',
|
||||
user_id,
|
||||
username,
|
||||
display_name,
|
||||
password_hash,
|
||||
created,
|
||||
)
|
||||
except asyncpg.UniqueViolationError:
|
||||
raise conflict('Username уже занят') from None
|
||||
return User(str(user_id), username, display_name, _stamp(created))
|
||||
|
||||
async def authenticate(self, username: str, password: str) -> Optional[User]:
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT id, username, display_name, created_at, password_hash
|
||||
FROM users
|
||||
WHERE username = $1
|
||||
''',
|
||||
username,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
stored_hash = row['password_hash'] if row is not None else None
|
||||
if not password_matches(stored_hash, password) or row is None:
|
||||
return None
|
||||
return _user(row)
|
||||
|
||||
async def create_session(self, user_id: str) -> Tuple[str, str]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires = datetime.now(timezone.utc) + self._session_ttl
|
||||
key = _token_key(token)
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES ($1, $2, $3)
|
||||
''',
|
||||
key,
|
||||
_as_uuid(user_id, 'Некорректный user_id'),
|
||||
expires,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
await self._cache.remember(key, expires, datetime.now(timezone.utc), only_if_absent=False)
|
||||
return token, _stamp(expires)
|
||||
|
||||
async def user_for_token(self, token: str) -> Optional[User]:
|
||||
now = datetime.now(timezone.utc)
|
||||
key = _token_key(token)
|
||||
cached = await self._cache.get(key)
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
|
||||
FROM sessions s
|
||||
JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1
|
||||
''',
|
||||
key,
|
||||
)
|
||||
if row is None or row['expires_at'] <= now:
|
||||
if row is not None:
|
||||
await conn.execute('DELETE FROM sessions WHERE token_hash = $1', key)
|
||||
await self._cache.drop(key)
|
||||
return None
|
||||
if cached is None:
|
||||
await self._cache.remember(key, row['expires_at'], now, only_if_absent=True)
|
||||
return _user(row)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
|
||||
async def revoke_session(self, token: str) -> bool:
|
||||
now = datetime.now(timezone.utc)
|
||||
key = _token_key(token)
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
DELETE FROM sessions
|
||||
WHERE token_hash = $1 AND expires_at > $2
|
||||
RETURNING token_hash
|
||||
''',
|
||||
key,
|
||||
now,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
await self._cache.drop(key)
|
||||
return row is not None
|
||||
|
||||
async def create_chat(self, title: str, member_ids: Sequence[str]) -> Chat:
|
||||
ids = [_as_uuid(member_id, 'Некорректный user_id') for member_id in member_ids]
|
||||
chat_id = uuid.uuid4()
|
||||
created = self._now()
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
found = await conn.fetch(
|
||||
'SELECT id FROM users WHERE id = ANY($1::uuid[])',
|
||||
ids,
|
||||
)
|
||||
if len(found) != len(ids):
|
||||
raise invalid_request('Все участники должны быть существующими пользователями')
|
||||
await conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)',
|
||||
chat_id,
|
||||
title,
|
||||
created,
|
||||
)
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT $1, member_id, (ordinality - 1)::integer
|
||||
FROM unnest($2::uuid[]) WITH ORDINALITY AS members(member_id, ordinality)
|
||||
''',
|
||||
chat_id,
|
||||
ids,
|
||||
)
|
||||
return Chat(str(chat_id), title, tuple(member_ids), _stamp(created))
|
||||
|
||||
async def find_chat(self, chat_id: str, member_id: str) -> Optional[Chat]:
|
||||
"""A chat of somebody else is indistinguishable from a missing one."""
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
member_uuid = _as_uuid(member_id, 'Некорректный user_id')
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2
|
||||
WHERE c.id = $1
|
||||
''',
|
||||
chat_uuid,
|
||||
member_uuid,
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
members = await self._members(conn, [chat_uuid])
|
||||
return Chat(str(row['id']), row['title'], members.get(chat_uuid, ()), _stamp(row['created_at']))
|
||||
|
||||
async def list_chats(self, member_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Chat], bool]:
|
||||
member_uuid = _as_uuid(member_id, 'Некорректный user_id')
|
||||
after_at, after_id = self._cursor(after)
|
||||
async with self._pool.acquire() as conn:
|
||||
if after_at is None:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $2
|
||||
''',
|
||||
member_uuid,
|
||||
limit + 1,
|
||||
)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1
|
||||
WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $4
|
||||
''',
|
||||
member_uuid,
|
||||
after_at,
|
||||
after_id,
|
||||
limit + 1,
|
||||
)
|
||||
page = rows[:limit]
|
||||
members = await self._members(conn, [row['id'] for row in page])
|
||||
chats = [
|
||||
Chat(str(row['id']), row['title'], members.get(row['id'], ()), _stamp(row['created_at']))
|
||||
for row in page
|
||||
]
|
||||
return chats, len(rows) > limit
|
||||
|
||||
async def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
|
||||
message_id = uuid.uuid4()
|
||||
created = self._now()
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
version = await conn.fetchval(
|
||||
'''
|
||||
UPDATE chats
|
||||
SET history_version = history_version + 1
|
||||
WHERE id = $1
|
||||
RETURNING history_version
|
||||
''',
|
||||
chat_uuid,
|
||||
)
|
||||
if version is None:
|
||||
raise not_found('Чат не найден')
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
''',
|
||||
message_id,
|
||||
chat_uuid,
|
||||
_as_uuid(sender_id, 'Некорректный user_id'),
|
||||
text,
|
||||
created,
|
||||
)
|
||||
if self._history_mode == 'protected':
|
||||
await self._cache.publish_history_version(str(chat_uuid), int(version))
|
||||
return Message(str(message_id), str(chat_uuid), sender_id, text, _stamp(created))
|
||||
|
||||
async def list_messages(self, chat_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Message], bool]:
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
after_at, after_id = self._cursor(after)
|
||||
if after_at is None and limit == _FIRST_PAGE_LIMIT and self._history_mode != 'off':
|
||||
if self._history_mode == 'protected':
|
||||
return await self._protected_first_page(chat_uuid, limit)
|
||||
return await self._naive_first_page(chat_uuid, limit)
|
||||
return await self._fetch_messages(chat_uuid, limit, after_at, after_id)
|
||||
|
||||
async def _protected_first_page(self, chat_uuid: uuid.UUID, limit: int) -> Tuple[List[Message], bool]:
|
||||
chat_id = str(chat_uuid)
|
||||
version = await self._cache.history_version(chat_id)
|
||||
if version is None:
|
||||
version = await self._load_history_version(chat_uuid)
|
||||
await self._cache.publish_history_version(chat_id, version)
|
||||
key = 'history:%s:%s' % (chat_id, version)
|
||||
cached = _decode_page(await self._cache.get_history_page(key))
|
||||
if cached is not None:
|
||||
return cached
|
||||
return await self._singleflight(key, lambda: self._fill_page(key, chat_uuid, limit))
|
||||
|
||||
async def _naive_first_page(self, chat_uuid: uuid.UUID, limit: int) -> Tuple[List[Message], bool]:
|
||||
key = 'history:%s' % chat_uuid
|
||||
cached = _decode_page(await self._cache.get_history_page(key))
|
||||
if cached is not None:
|
||||
return cached
|
||||
messages, has_more = await self._fetch_messages(chat_uuid, limit, None, None)
|
||||
await asyncio.sleep(_NAIVE_FILL_HOLD_SECONDS)
|
||||
await self._cache.set_history_page(key, _encode_page(messages, has_more))
|
||||
return messages, has_more
|
||||
|
||||
async def _fill_page(self, key: str, chat_uuid: uuid.UUID, limit: int) -> Tuple[List[Message], bool]:
|
||||
cached = _decode_page(await self._cache.get_history_page(key))
|
||||
if cached is not None:
|
||||
return cached
|
||||
messages, has_more = await self._fetch_messages(chat_uuid, limit, None, None)
|
||||
await self._cache.set_history_page(key, _encode_page(messages, has_more))
|
||||
return messages, has_more
|
||||
|
||||
async def _singleflight(self, key: str, loader):
|
||||
async with self._flight_lock:
|
||||
flight = self._flights.get(key)
|
||||
if flight is None:
|
||||
flight = asyncio.get_running_loop().create_future()
|
||||
self._flights[key] = flight
|
||||
leader = True
|
||||
else:
|
||||
leader = False
|
||||
if not leader:
|
||||
return await flight
|
||||
try:
|
||||
value = await loader()
|
||||
except Exception as exc:
|
||||
if not flight.done():
|
||||
flight.set_exception(exc)
|
||||
raise
|
||||
else:
|
||||
if not flight.done():
|
||||
flight.set_result(value)
|
||||
return value
|
||||
finally:
|
||||
async with self._flight_lock:
|
||||
if self._flights.get(key) is flight:
|
||||
del self._flights[key]
|
||||
|
||||
async def _load_history_version(self, chat_uuid: uuid.UUID) -> int:
|
||||
async with self._pool.acquire() as conn:
|
||||
version = await conn.fetchval(
|
||||
'SELECT history_version FROM chats WHERE id = $1',
|
||||
chat_uuid,
|
||||
)
|
||||
if version is None:
|
||||
raise not_found('Чат не найден')
|
||||
return int(version)
|
||||
|
||||
async def _fetch_messages(
|
||||
self,
|
||||
chat_uuid: uuid.UUID,
|
||||
limit: int,
|
||||
after_at: Optional[datetime],
|
||||
after_id: Optional[uuid.UUID],
|
||||
) -> Tuple[List[Message], bool]:
|
||||
async with self._pool.acquire() as conn:
|
||||
if after_at is None:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2
|
||||
''',
|
||||
chat_uuid,
|
||||
limit + 1,
|
||||
)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND (created_at, id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4
|
||||
''',
|
||||
chat_uuid,
|
||||
after_at,
|
||||
after_id,
|
||||
limit + 1,
|
||||
)
|
||||
page = rows[:limit]
|
||||
messages = [
|
||||
Message(str(row['id']), str(row['chat_id']), str(row['sender_id']), row['text'], _stamp(row['created_at']))
|
||||
for row in page
|
||||
]
|
||||
return messages, len(rows) > limit
|
||||
|
||||
async def _members(self, conn: asyncpg.Connection, chat_ids: Sequence[uuid.UUID]) -> Dict[uuid.UUID, Tuple[str, ...]]:
|
||||
if not chat_ids:
|
||||
return {}
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT chat_id, user_id
|
||||
FROM chat_members
|
||||
WHERE chat_id = ANY($1::uuid[])
|
||||
ORDER BY chat_id, position
|
||||
''',
|
||||
list(chat_ids),
|
||||
)
|
||||
grouped: Dict[uuid.UUID, List[str]] = {}
|
||||
for row in rows:
|
||||
grouped.setdefault(row['chat_id'], []).append(str(row['user_id']))
|
||||
return {chat_id: tuple(member_ids) for chat_id, member_ids in grouped.items()}
|
||||
|
||||
def _cursor(self, after: Optional[Key]) -> Tuple[Optional[datetime], Optional[uuid.UUID]]:
|
||||
if after is None:
|
||||
return None, None
|
||||
try:
|
||||
return _parse_stamp(after[0]), uuid.UUID(after[1])
|
||||
except ValueError:
|
||||
raise invalid_request('Некорректный cursor') from None
|
||||
@@ -0,0 +1,67 @@
|
||||
# Local stand. Secrets come from .env, which is not committed.
|
||||
# PostgreSQL is the source of truth. Redis caches session lookups and has no volume.
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.10-alpine
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
ports:
|
||||
- "127.0.0.1:5432:5432"
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7.4.11-alpine
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no"]
|
||||
ports:
|
||||
- "127.0.0.1:6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
HISTORY_CACHE: ${HISTORY_CACHE:-protected}
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
@@ -0,0 +1,16 @@
|
||||
rows_in_chat: 10000
|
||||
query: keyset page after the 50th message, LIMIT 51
|
||||
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (cost=0.29..4.19 rows=51 width=58) (actual time=0.047..0.070 rows=51 loops=1)
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using messages_history_idx on messages (cost=0.29..762.42 rows=9950 width=58) (actual time=0.044..0.063 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '49b53d26-7fad-40e0-8811-59eae519e639'::uuid) AND (ROW(created_at, id) > ROW('2024-01-01 00:00:00.00005+00'::timestamp with time zone, '38a04f80-174d-4233-ac1b-f319ea7668f3'::uuid)))
|
||||
Buffers: shared hit=4
|
||||
Planning:
|
||||
Buffers: shared hit=106
|
||||
Planning Time: 0.886 ms
|
||||
Execution Time: 0.363 ms
|
||||
(9 rows)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
Grade 3 checks. No tokens, passwords, or connection strings.
|
||||
|
||||
make check equivalent:
|
||||
.venv/Scripts/python.exe scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
Pinned image smoke, http://127.0.0.1:8080:
|
||||
.venv/Scripts/python.exe tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
Persistence on the same image. The public harness executes scripts/restart.sh.
|
||||
On this Windows host CreateProcess cannot run a shell script (WinError 193), and the
|
||||
only WSL distro is docker-desktop. The harness was therefore run inside docker:28-cli
|
||||
with python3, the compose network, and the Docker socket. The hook was still
|
||||
scripts/restart.sh. BASE_URL was http://api:8080.
|
||||
PASS: persistence preserved confirmed data and existing session.
|
||||
|
||||
Containers before and after that hook kept the same IDs (restart, not recreate):
|
||||
api 15bfccf88c7a
|
||||
postgres cbe7139645a5
|
||||
Volume still present: mtusi2026lab2-fork_pgdata
|
||||
|
||||
Second API start, same volume, migrations not re-inserted:
|
||||
before: migrations=1 users=14 messages=6
|
||||
after: migrations=1 users=14 messages=6
|
||||
|
||||
Unknown member:
|
||||
.venv/Scripts/python.exe scripts/check_grade3.py
|
||||
PASS: unknown member returns 400 and leaves no chat row
|
||||
|
||||
Short TTL, separate from persistence. API recreated with SESSION_TTL_SECONDS=2,
|
||||
then .env restored to 3600 and the API recreated again.
|
||||
PASS: same token got 401 after 3 seconds.
|
||||
|
||||
Postgres stopped, API left running:
|
||||
GET /health/live 200 {"status":"ok"}
|
||||
GET /health/ready 503 {"error":{"code":"unavailable","message":"Хранилище недоступно",...}}
|
||||
request_id in the body matched X-Request-Id. Body had no SQL.
|
||||
Postgres was started again; /health/ready returned 200.
|
||||
@@ -0,0 +1,31 @@
|
||||
Grade 4 checks. No tokens or passwords.
|
||||
|
||||
python scripts/check_grade4.py
|
||||
PASS: one 201 and seven 409 for the same username
|
||||
PASS: unknown member rolls back chat and membership
|
||||
PASS: wrong password and unknown user share the same 401 challenge
|
||||
PASS: logout returns 204, then 401, and drops the cache key
|
||||
PASS: Redis outage falls back to PostgreSQL and still rejects anonymous requests
|
||||
PASS: PostgreSQL outage is 503 for a bearer and 401 without one
|
||||
PASS: SESSION_TTL_SECONDS=2 rejects the same token after 3 seconds
|
||||
PASS: grade 4 checks
|
||||
|
||||
After that script the API environment was SESSION_TTL_SECONDS=3600.
|
||||
|
||||
python scripts/explain_history.py
|
||||
PASS: history index used at 10000 messages
|
||||
Plan: Index Scan using messages_history_idx, LIMIT 51, Buffers shared hit=4, no Seq Scan.
|
||||
Saved as evidence/explain.txt. The seed chat was deleted afterwards.
|
||||
|
||||
Smoke on the rebuilt image:
|
||||
PASS: lab2 public API scenario
|
||||
|
||||
Persistence, hook scripts/restart.sh restarts api, postgres and redis.
|
||||
Ran inside docker:28-cli because Windows CreateProcess cannot execute the shell script.
|
||||
PASS: persistence preserved confirmed data and existing session.
|
||||
Container IDs unchanged:
|
||||
api 87a78974fbc8
|
||||
postgres cbe7139645a5
|
||||
redis e271b1f6ea0e
|
||||
Volume still present: mtusi2026lab2-fork_pgdata
|
||||
Redis has no volume; the session row in PostgreSQL is what survived.
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"track": 1,
|
||||
"cached_limit": 50,
|
||||
"page_ttl_seconds": 30,
|
||||
"naive_fill_hold_seconds": 0.2,
|
||||
"naive": {
|
||||
"reads_started_after_201": 8,
|
||||
"reads_missing_new_message": 8,
|
||||
"stale_read_observed": true,
|
||||
"cold": {
|
||||
"clients": 32,
|
||||
"message_scans": 32,
|
||||
"p50_ms": 343.766
|
||||
}
|
||||
},
|
||||
"off": {
|
||||
"clients": 20,
|
||||
"message_scans": 20,
|
||||
"p50_ms": 4.95
|
||||
},
|
||||
"protected": {
|
||||
"reads_started_after_201": 8,
|
||||
"reads_missing_new_message": 0,
|
||||
"read_after_write": true,
|
||||
"cold": {
|
||||
"clients": 32,
|
||||
"message_scans": 1,
|
||||
"p50_ms": 160.808
|
||||
},
|
||||
"sequential_miss_p50_ms": 8.519,
|
||||
"warm": {
|
||||
"clients": 20,
|
||||
"message_scans": 0,
|
||||
"p50_ms": 5.786
|
||||
},
|
||||
"after_redis_restart": {
|
||||
"confirmed_message_present": true,
|
||||
"first_read_message_scans": 1,
|
||||
"second_read_message_scans": 0,
|
||||
"session_status": 200
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
-- Initial schema. The runner records this file in schema_migrations and skips it on the next start.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT users_username_unique UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
expires_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
CONSTRAINT chat_members_position_nonnegative CHECK (position >= 0)
|
||||
);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
sender_id UUID NOT NULL REFERENCES users (id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
);
|
||||
|
||||
-- Keyset history: WHERE chat_id = ? AND (created_at, id) > cursor ORDER BY created_at, id.
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Chat list is filtered by the caller, then ordered by the chat's own (created_at, id).
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE chats
|
||||
ADD COLUMN history_version BIGINT NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,20 @@
|
||||
# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned.
|
||||
asyncpg==0.32.0
|
||||
redis==8.1.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==25.1.0
|
||||
cffi==2.0.0
|
||||
click==8.5.0
|
||||
fastapi==0.141.1
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
pycparser==3.0
|
||||
pydantic==2.13.5
|
||||
pydantic_core==2.46.5
|
||||
starlette==1.7.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
uvicorn==0.53.0
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-3 check that is not part of make test.
|
||||
|
||||
Creates a chat with one unknown member and expects 400. Then asks PostgreSQL
|
||||
whether that title was stored. A rollback leaves zero rows.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
|
||||
|
||||
def request(method, path, body=None, headers=None, expected=200):
|
||||
data = None if body is None else json.dumps(body).encode('utf-8')
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, method=method)
|
||||
req.add_header('Content-Type', 'application/json')
|
||||
for key, value in (headers or {}).items():
|
||||
req.add_header(key, value)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
status = response.status
|
||||
payload = response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
status = exc.code
|
||||
payload = exc.read()
|
||||
if status != expected:
|
||||
sys.stderr.write('expected %s %s -> %s, got %s %s\n' % (method, path, expected, status, payload[:300]))
|
||||
return None
|
||||
if not payload:
|
||||
return {}
|
||||
return json.loads(payload)
|
||||
|
||||
|
||||
def main():
|
||||
username = 'u_orphan_' + uuid.uuid4().hex[:12]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
user = request('POST', '/api/v1/users', {
|
||||
'username': username,
|
||||
'display_name': 'Orphan check',
|
||||
'password': password,
|
||||
}, expected=201)
|
||||
if user is None:
|
||||
return 1
|
||||
import base64
|
||||
basic = base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii')
|
||||
session = request('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic}, expected=201)
|
||||
if session is None:
|
||||
return 1
|
||||
title = 'orphan-' + uuid.uuid4().hex
|
||||
missing = str(uuid.uuid4())
|
||||
rejected = request(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
{'title': title, 'member_ids': [user['id'], missing]},
|
||||
headers={'Authorization': 'Bearer ' + session['token']},
|
||||
expected=400,
|
||||
)
|
||||
if rejected is None:
|
||||
return 1
|
||||
sql = "SELECT count(*) FROM chats WHERE title = '%s'" % title.replace("'", '')
|
||||
probe = subprocess.run(
|
||||
[
|
||||
'docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||||
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
||||
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
||||
'-tAc', sql,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if probe.returncode != 0:
|
||||
sys.stderr.write(probe.stderr)
|
||||
return 1
|
||||
count = probe.stdout.strip()
|
||||
if count != '0':
|
||||
sys.stderr.write('rolled-back chat is still stored: count=%s\n' % count)
|
||||
return 1
|
||||
print('PASS: unknown member returns 400 and leaves no chat row')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-4 checks. Not part of make test. Restores Redis, PostgreSQL and SESSION_TTL."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
ENV_PATH = ROOT / '.env'
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, timeout=20):
|
||||
data = None if body is None else json.dumps(body).encode('utf-8')
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, method=method)
|
||||
if body is not None:
|
||||
req.add_header('Content-Type', 'application/json')
|
||||
for key, value in (headers or {}).items():
|
||||
req.add_header(key, value)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
payload = response.read()
|
||||
return response.status, payload, response.headers
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read(), exc.headers
|
||||
|
||||
|
||||
def compose(*args):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', *args],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('docker compose %s failed' % ' '.join(args))
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def psql(sql):
|
||||
proc = subprocess.run(
|
||||
[
|
||||
'docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||||
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
||||
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
||||
'-v', 'ON_ERROR_STOP=1', '-tAc', sql,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('psql failed')
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def redis_cli(*args):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'redis', 'redis-cli', *args],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('redis-cli failed')
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def wait_ready(seconds=45):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, _, _ = call('GET', '/health/ready', timeout=3)
|
||||
if status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise SystemExit('API did not become ready')
|
||||
|
||||
|
||||
def register(label):
|
||||
username = 'u_%s_%s' % (label, uuid.uuid4().hex[:12])
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
status, raw, _ = call('POST', '/api/v1/users', {
|
||||
'username': username,
|
||||
'display_name': label,
|
||||
'password': password,
|
||||
})
|
||||
if status != 201:
|
||||
raise SystemExit('register failed: %s' % status)
|
||||
user = json.loads(raw)
|
||||
return user, username, password
|
||||
|
||||
|
||||
def login(username, password):
|
||||
basic = base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii')
|
||||
status, raw, headers = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic})
|
||||
return status, raw, headers
|
||||
|
||||
|
||||
def bearer(token):
|
||||
return {'Authorization': 'Bearer ' + token}
|
||||
|
||||
|
||||
def session_key(token):
|
||||
return 'session:' + hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def check_race():
|
||||
username = 'u_race_' + uuid.uuid4().hex[:12]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
body = {'username': username, 'display_name': 'Race', 'password': password}
|
||||
|
||||
def once():
|
||||
status, _, _ = call('POST', '/api/v1/users', body)
|
||||
return status
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
statuses = list(pool.map(lambda _: once(), range(8)))
|
||||
if statuses.count(201) != 1 or statuses.count(409) != 7:
|
||||
raise SystemExit('username race statuses: %s' % statuses)
|
||||
print('PASS: one 201 and seven 409 for the same username')
|
||||
|
||||
|
||||
def check_rollback():
|
||||
user, username, password = register('rollback')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('rollback login failed')
|
||||
token = json.loads(raw)['token']
|
||||
title = 'rollback-' + uuid.uuid4().hex
|
||||
missing = str(uuid.uuid4())
|
||||
status, _, _ = call(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
{'title': title, 'member_ids': [user['id'], missing]},
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 400:
|
||||
raise SystemExit('expected 400 for an unknown member, got %s' % status)
|
||||
safe = title.replace("'", '')
|
||||
chats = psql("SELECT count(*) FROM chats WHERE title = '%s'" % safe)
|
||||
members = psql(
|
||||
"SELECT count(*) FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE c.title = '%s'" % safe
|
||||
)
|
||||
if chats != '0' or members != '0':
|
||||
raise SystemExit('rollback left rows chats=%s members=%s' % (chats, members))
|
||||
print('PASS: unknown member rolls back chat and membership')
|
||||
|
||||
|
||||
def check_wrong_password():
|
||||
_, username, password = register('pw')
|
||||
bad_status, _, bad_headers = login(username, 'wrong-password-value')
|
||||
missing_status, _, missing_headers = login('missing_' + uuid.uuid4().hex[:8], password)
|
||||
bad_auth = bad_headers.get('WWW-Authenticate', '')
|
||||
missing_auth = missing_headers.get('WWW-Authenticate', '')
|
||||
if bad_status != 401 or missing_status != 401:
|
||||
raise SystemExit('password statuses %s %s' % (bad_status, missing_status))
|
||||
if not bad_auth.lower().startswith('basic') or bad_auth != missing_auth:
|
||||
raise SystemExit('WWW-Authenticate mismatch')
|
||||
print('PASS: wrong password and unknown user share the same 401 challenge')
|
||||
|
||||
|
||||
def check_logout():
|
||||
_, username, password = register('logout')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('logout login failed')
|
||||
token = json.loads(raw)['token']
|
||||
status, body, _ = call('DELETE', '/api/v1/auth/sessions/current', headers=bearer(token))
|
||||
if status != 204 or body:
|
||||
raise SystemExit('logout did not return an empty 204')
|
||||
status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
if status != 401:
|
||||
raise SystemExit('revoked token was accepted')
|
||||
if redis_cli('EXISTS', session_key(token)) != '0':
|
||||
raise SystemExit('revoked session key is still cached')
|
||||
print('PASS: logout returns 204, then 401, and drops the cache key')
|
||||
|
||||
|
||||
def check_redis_down():
|
||||
_, username, password = register('redis')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('redis-down login failed')
|
||||
token = json.loads(raw)['token']
|
||||
compose('stop', 'redis')
|
||||
try:
|
||||
status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
if status != 200:
|
||||
raise SystemExit('valid session failed while Redis was down: %s' % status)
|
||||
status, _, _ = call('GET', '/api/v1/users/me')
|
||||
if status != 401:
|
||||
raise SystemExit('anonymous request was not 401 while Redis was down: %s' % status)
|
||||
finally:
|
||||
compose('start', 'redis')
|
||||
wait_ready()
|
||||
print('PASS: Redis outage falls back to PostgreSQL and still rejects anonymous requests')
|
||||
|
||||
|
||||
def check_postgres_down():
|
||||
_, username, password = register('pg')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('postgres-down login failed')
|
||||
token = json.loads(raw)['token']
|
||||
compose('stop', 'postgres')
|
||||
try:
|
||||
status, body, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
text = body.decode('utf-8', 'replace').lower()
|
||||
if status != 503:
|
||||
raise SystemExit('expected 503 when PostgreSQL is down, got %s' % status)
|
||||
if 'asyncpg' in text or 'select ' in text or 'syntax' in text:
|
||||
raise SystemExit('503 body leaked database details')
|
||||
status, _, _ = call('GET', '/api/v1/users/me')
|
||||
if status != 401:
|
||||
raise SystemExit('anonymous request was not 401 while PostgreSQL was down: %s' % status)
|
||||
finally:
|
||||
compose('start', 'postgres')
|
||||
wait_ready()
|
||||
print('PASS: PostgreSQL outage is 503 for a bearer and 401 without one')
|
||||
|
||||
|
||||
def check_expiry():
|
||||
original = ENV_PATH.read_bytes()
|
||||
text = original.decode('utf-8')
|
||||
updated, count = re.subn(
|
||||
r'^SESSION_TTL_SECONDS=.*$',
|
||||
'SESSION_TTL_SECONDS=2',
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
if count != 1:
|
||||
raise SystemExit('SESSION_TTL_SECONDS is missing from .env')
|
||||
ENV_PATH.write_bytes(updated.encode('utf-8'))
|
||||
try:
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'api')
|
||||
wait_ready()
|
||||
_, username, password = register('ttl')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('ttl login failed')
|
||||
token = json.loads(raw)['token']
|
||||
time.sleep(3)
|
||||
status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
if status != 401:
|
||||
raise SystemExit('expired token was accepted: %s' % status)
|
||||
finally:
|
||||
ENV_PATH.write_bytes(original)
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'api')
|
||||
wait_ready()
|
||||
print('PASS: SESSION_TTL_SECONDS=2 rejects the same token after 3 seconds')
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
check_race()
|
||||
check_rollback()
|
||||
check_wrong_password()
|
||||
check_logout()
|
||||
check_redis_down()
|
||||
check_postgres_down()
|
||||
check_expiry()
|
||||
finally:
|
||||
subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'start', 'postgres', 'redis'],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
)
|
||||
print('PASS: grade 4 checks')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) for keyset history. Not part of make test.
|
||||
|
||||
Seeds one chat until the default planner uses messages_history_idx, then
|
||||
deletes that seed. The saved plan contains no tokens.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / 'evidence' / 'explain.txt'
|
||||
SIZES = (10000, 50000, 200000)
|
||||
|
||||
|
||||
def psql(sql, tuples_only=False):
|
||||
command = [
|
||||
'docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||||
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
||||
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
||||
'-v', 'ON_ERROR_STOP=1',
|
||||
]
|
||||
if tuples_only:
|
||||
command.append('-tA')
|
||||
proc = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
input=sql,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('psql failed')
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def seed(user_id, chat_id, count):
|
||||
psql(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(),
|
||||
'%(chat)s'::uuid,
|
||||
'%(user)s'::uuid,
|
||||
'm',
|
||||
timestamptz '2024-01-01 00:00:00+00' + (g * interval '1 microsecond')
|
||||
FROM generate_series(1, %(count)d) AS g;
|
||||
ANALYZE messages;
|
||||
""" % {'chat': chat_id, 'user': user_id, 'count': count}
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
user_id = str(uuid.uuid4())
|
||||
chat_id = str(uuid.uuid4())
|
||||
username = 'u_explain_' + uuid.uuid4().hex[:12]
|
||||
psql(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ('%(user)s', '%(name)s', 'Explain', 'not-used-for-login', clock_timestamp());
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
VALUES ('%(chat)s', 'explain-seed', clock_timestamp());
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
VALUES ('%(chat)s', '%(user)s', 0);
|
||||
""" % {'user': user_id, 'chat': chat_id, 'name': username}
|
||||
)
|
||||
plan = ''
|
||||
used = 0
|
||||
try:
|
||||
for count in SIZES:
|
||||
seed(user_id, chat_id, count)
|
||||
used += count
|
||||
cursor = psql(
|
||||
"""
|
||||
SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), id::text
|
||||
FROM messages
|
||||
WHERE chat_id = '%s'::uuid
|
||||
ORDER BY created_at, id
|
||||
OFFSET 49
|
||||
LIMIT 1;
|
||||
""" % chat_id,
|
||||
tuples_only=True,
|
||||
).strip()
|
||||
created_at, message_id = cursor.split('|')
|
||||
plan = psql(
|
||||
"""
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = '%(chat)s'::uuid
|
||||
AND (created_at, id) > ('%(created)s+00'::timestamptz, '%(message)s'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
""" % {'chat': chat_id, 'created': created_at, 'message': message_id}
|
||||
)
|
||||
if 'Index Scan using messages_history_idx' in plan or 'Index Only Scan using messages_history_idx' in plan:
|
||||
break
|
||||
sys.stderr.write('planner did not pick the history index at %d rows\n' % used)
|
||||
else:
|
||||
sys.stderr.write(plan)
|
||||
raise SystemExit('history query still does not use messages_history_idx')
|
||||
if 'Seq Scan' in plan.split('Index Scan', 1)[0] and 'messages_history_idx' not in plan.split('Index Scan', 1)[0]:
|
||||
pass
|
||||
EVIDENCE.parent.mkdir(exist_ok=True)
|
||||
EVIDENCE.write_text(
|
||||
'rows_in_chat: %d\nquery: keyset page after the 50th message, LIMIT 51\n\n%s' % (used, plan),
|
||||
encoding='utf-8',
|
||||
newline='\n',
|
||||
)
|
||||
print('PASS: history index used at %d messages; wrote %s' % (used, EVIDENCE.name))
|
||||
finally:
|
||||
psql(
|
||||
"""
|
||||
DELETE FROM messages WHERE chat_id = '%(chat)s';
|
||||
DELETE FROM chat_members WHERE chat_id = '%(chat)s';
|
||||
DELETE FROM chats WHERE id = '%(chat)s';
|
||||
DELETE FROM users WHERE id = '%(user)s';
|
||||
""" % {'chat': chat_id, 'user': user_id}
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Restart the API, PostgreSQL and the session cache. Named volumes stay; nothing is re-seeded.
|
||||
docker compose --env-file .env restart api postgres redis
|
||||
@@ -0,0 +1,448 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1: stale first page, then versioned cache and singleflight.
|
||||
|
||||
The script switches HISTORY_CACHE, recreates only the API, and always restores
|
||||
the previous .env. It does not print tokens.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
ENV_PATH = ROOT / '.env'
|
||||
EVIDENCE_PATH = ROOT / 'evidence' / 'track1.json'
|
||||
COLD_CLIENTS = 32
|
||||
READERS = 8
|
||||
WARM_READS = 20
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, timeout=20):
|
||||
data = None if body is None else json.dumps(body).encode('utf-8')
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, method=method)
|
||||
if body is not None:
|
||||
req.add_header('Content-Type', 'application/json')
|
||||
for key, value in (headers or {}).items():
|
||||
req.add_header(key, value)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
payload = response.read()
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
return response.status, payload, elapsed_ms
|
||||
except urllib.error.HTTPError as exc:
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
return exc.code, exc.read(), elapsed_ms
|
||||
|
||||
|
||||
def compose(*args, check=True):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', *args],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if check and proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('docker compose %s failed' % ' '.join(args))
|
||||
return proc
|
||||
|
||||
|
||||
def psql(sql):
|
||||
proc = compose(
|
||||
'exec', '-T', 'postgres',
|
||||
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
||||
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
||||
'-v', 'ON_ERROR_STOP=1', '-tAc', sql,
|
||||
)
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def redis_cli(*args, check=True):
|
||||
proc = compose('exec', '-T', 'redis', 'redis-cli', *args, check=check)
|
||||
return proc
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, _, _ = call('GET', '/health/ready', timeout=3)
|
||||
if status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise SystemExit('API did not become ready')
|
||||
|
||||
|
||||
def wait_redis(seconds=40):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
proc = redis_cli('ping', check=False)
|
||||
if proc.returncode == 0 and 'PONG' in proc.stdout:
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise SystemExit('redis did not answer PING')
|
||||
|
||||
|
||||
def message_scans():
|
||||
raw = psql(
|
||||
"SELECT COALESCE(idx_scan,0)+COALESCE(seq_scan,0) "
|
||||
"FROM pg_stat_user_tables WHERE relname = 'messages'"
|
||||
)
|
||||
if not raw:
|
||||
raise SystemExit('messages stats are missing')
|
||||
return int(raw)
|
||||
|
||||
|
||||
def drained_message_scans():
|
||||
"""Backends publish table stats on exit. Drop the ones that ran the workload."""
|
||||
raw = psql(
|
||||
"SELECT pid FROM pg_stat_activity "
|
||||
"WHERE datname = current_database() AND pid <> pg_backend_pid() "
|
||||
"AND backend_type = 'client backend'"
|
||||
)
|
||||
pids = [line.strip() for line in raw.splitlines() if line.strip()]
|
||||
if pids:
|
||||
if not all(pid.isdigit() for pid in pids):
|
||||
raise SystemExit('unexpected pid list')
|
||||
listed = ','.join(pids)
|
||||
psql('SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid IN (%s)' % listed)
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
still = psql('SELECT count(*) FROM pg_stat_activity WHERE pid IN (%s)' % listed)
|
||||
if still == '0':
|
||||
break
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
raise SystemExit('postgres backends did not disconnect')
|
||||
return message_scans()
|
||||
|
||||
|
||||
def p50(samples):
|
||||
ordered = sorted(samples)
|
||||
return ordered[(len(ordered) - 1) // 2]
|
||||
|
||||
|
||||
def bearer(token):
|
||||
return {'Authorization': 'Bearer ' + token}
|
||||
|
||||
|
||||
def write_mode(mode):
|
||||
text = ENV_PATH.read_text(encoding='utf-8')
|
||||
newline = '\r\n' if '\r\n' in text else '\n'
|
||||
lines = [line for line in text.splitlines() if not line.startswith('HISTORY_CACHE=')]
|
||||
lines.append('HISTORY_CACHE=' + mode)
|
||||
ENV_PATH.write_bytes((newline.join(lines) + newline).encode('utf-8'))
|
||||
|
||||
|
||||
def recreate_api(build=False):
|
||||
args = ['up', '-d', '--force-recreate', '--no-deps']
|
||||
if build:
|
||||
args = ['up', '-d', '--build', '--force-recreate', '--no-deps']
|
||||
args.append('api')
|
||||
compose(*args)
|
||||
wait_ready()
|
||||
|
||||
|
||||
def flush_history():
|
||||
keys = []
|
||||
for pattern in ('history:*', 'hver:*'):
|
||||
proc = redis_cli('--scan', '--pattern', pattern)
|
||||
keys.extend(line.strip() for line in proc.stdout.splitlines() if line.strip())
|
||||
if keys:
|
||||
redis_cli('DEL', *keys)
|
||||
|
||||
|
||||
def register_and_chat():
|
||||
username = 'u_t1_%s' % uuid.uuid4().hex[:12]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
status, raw, _ = call('POST', '/api/v1/users', {
|
||||
'username': username,
|
||||
'display_name': 'track1',
|
||||
'password': password,
|
||||
})
|
||||
if status != 201:
|
||||
raise SystemExit('register failed: %s' % status)
|
||||
user = json.loads(raw)
|
||||
basic = base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii')
|
||||
status, raw, _ = call(
|
||||
'POST', '/api/v1/auth/sessions',
|
||||
headers={'Authorization': 'Basic ' + basic},
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit('login failed: %s' % status)
|
||||
token = json.loads(raw)['token']
|
||||
status, raw, _ = call(
|
||||
'POST', '/api/v1/chats',
|
||||
{'title': 'track1', 'member_ids': [user['id']]},
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit('chat failed: %s' % status)
|
||||
chat_id = json.loads(raw)['id']
|
||||
status, _, _ = call(
|
||||
'POST', '/api/v1/chats/%s/messages' % chat_id,
|
||||
{'text': 'seed'},
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit('seed message failed: %s' % status)
|
||||
return token, chat_id
|
||||
|
||||
|
||||
def reads_after_write(token, chat_id):
|
||||
status, _, _ = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 200:
|
||||
raise SystemExit('warm read failed: %s' % status)
|
||||
post_done = threading.Event()
|
||||
posted = {}
|
||||
|
||||
def poster():
|
||||
status, raw, _ = call(
|
||||
'POST', '/api/v1/chats/%s/messages' % chat_id,
|
||||
{'text': 'confirmed-' + uuid.uuid4().hex[:8]},
|
||||
headers=bearer(token),
|
||||
)
|
||||
posted['status'] = status
|
||||
if status == 201:
|
||||
posted['id'] = json.loads(raw)['id']
|
||||
post_done.set()
|
||||
|
||||
def reader(_index):
|
||||
if not post_done.wait(20):
|
||||
return None
|
||||
status, raw, _ = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 200:
|
||||
return None
|
||||
ids = [item['id'] for item in json.loads(raw)['items']]
|
||||
return posted.get('id') in ids
|
||||
|
||||
with ThreadPoolExecutor(max_workers=READERS + 1) as pool:
|
||||
post = pool.submit(poster)
|
||||
jobs = [pool.submit(reader, index) for index in range(READERS)]
|
||||
post.result()
|
||||
seen = [job.result() for job in jobs]
|
||||
if posted.get('status') != 201 or 'id' not in posted:
|
||||
raise SystemExit('post failed: %s' % posted.get('status'))
|
||||
present = sum(1 for item in seen if item is True)
|
||||
missing = sum(1 for item in seen if item is False)
|
||||
return {
|
||||
'reads_started_after_201': present + missing,
|
||||
'reads_missing_new_message': missing,
|
||||
'message_id': posted['id'],
|
||||
}
|
||||
|
||||
|
||||
def cold_reads(token, chat_id):
|
||||
flush_history()
|
||||
before = drained_message_scans()
|
||||
timings = []
|
||||
failures = []
|
||||
|
||||
def one(_index):
|
||||
status, _, elapsed = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
timings.append(elapsed)
|
||||
if status != 200:
|
||||
failures.append(status)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=COLD_CLIENTS) as pool:
|
||||
list(pool.map(one, range(COLD_CLIENTS)))
|
||||
if failures:
|
||||
raise SystemExit('cold reads failed: %s' % failures[:3])
|
||||
return {
|
||||
'clients': COLD_CLIENTS,
|
||||
'message_scans': drained_message_scans() - before,
|
||||
'p50_ms': round(p50(timings), 3),
|
||||
}
|
||||
|
||||
|
||||
def miss_latency(token, chat_id, samples=12):
|
||||
"""One GET after a flush. The clock starts at the request, not at the flush."""
|
||||
timings = []
|
||||
for _ in range(samples):
|
||||
flush_history()
|
||||
status, _, elapsed = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 200:
|
||||
raise SystemExit('cold miss failed: %s' % status)
|
||||
timings.append(elapsed)
|
||||
return round(p50(timings), 3)
|
||||
|
||||
|
||||
def warm_reads(token, chat_id):
|
||||
status, _, _ = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 200:
|
||||
raise SystemExit('warm fill failed: %s' % status)
|
||||
before = drained_message_scans()
|
||||
timings = []
|
||||
for _ in range(WARM_READS):
|
||||
status, _, elapsed = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 200:
|
||||
raise SystemExit('warm read failed: %s' % status)
|
||||
timings.append(elapsed)
|
||||
return {
|
||||
'clients': WARM_READS,
|
||||
'message_scans': drained_message_scans() - before,
|
||||
'p50_ms': round(p50(timings), 3),
|
||||
}
|
||||
|
||||
|
||||
def redis_restart(token, chat_id, message_id):
|
||||
compose('exec', '-T', 'redis', 'sh', '-c', 'rm -f /data/dump.rdb /data/appendonly.aof')
|
||||
compose('restart', 'redis')
|
||||
wait_redis()
|
||||
time.sleep(0.5)
|
||||
session_status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
before = drained_message_scans()
|
||||
status, raw, _ = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
first_delta = drained_message_scans() - before
|
||||
present = False
|
||||
if status == 200:
|
||||
present = message_id in [item['id'] for item in json.loads(raw)['items']]
|
||||
before = drained_message_scans()
|
||||
status, _, _ = call(
|
||||
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
||||
headers=bearer(token),
|
||||
)
|
||||
second_delta = drained_message_scans() - before
|
||||
return {
|
||||
'confirmed_message_present': present and status == 200,
|
||||
'first_read_message_scans': first_delta,
|
||||
'second_read_message_scans': second_delta,
|
||||
'session_status': session_status,
|
||||
}
|
||||
|
||||
|
||||
def problems_of(evidence):
|
||||
found = []
|
||||
naive = evidence['naive']
|
||||
protected = evidence['protected']
|
||||
if naive['reads_missing_new_message'] < 1:
|
||||
found.append('naive did not show a stale read')
|
||||
if protected['reads_missing_new_message'] != 0:
|
||||
found.append('protected missed a confirmed message')
|
||||
if evidence['off']['message_scans'] != WARM_READS:
|
||||
found.append('off scans %s' % evidence['off']['message_scans'])
|
||||
if naive['cold']['message_scans'] < COLD_CLIENTS - 8:
|
||||
found.append('naive cold scans %s' % naive['cold']['message_scans'])
|
||||
if protected['cold']['message_scans'] > 2:
|
||||
found.append('protected cold scans %s' % protected['cold']['message_scans'])
|
||||
if protected['warm']['message_scans'] != 0:
|
||||
found.append('warm scans %s' % protected['warm']['message_scans'])
|
||||
if protected['warm']['p50_ms'] >= protected['sequential_miss_p50_ms']:
|
||||
found.append('warm p50 is not below a cold miss')
|
||||
restart = protected['after_redis_restart']
|
||||
if not restart['confirmed_message_present']:
|
||||
found.append('message missing after redis restart')
|
||||
if restart['first_read_message_scans'] < 1:
|
||||
found.append('restart was not a miss')
|
||||
if restart['second_read_message_scans'] != 0:
|
||||
found.append('second read after restart was not a hit')
|
||||
if restart['session_status'] != 200:
|
||||
found.append('session after redis restart: %s' % restart['session_status'])
|
||||
return found
|
||||
|
||||
|
||||
def main():
|
||||
if not ENV_PATH.is_file():
|
||||
raise SystemExit('.env is missing')
|
||||
original = ENV_PATH.read_bytes()
|
||||
failure = None
|
||||
try:
|
||||
print('naive mode', flush=True)
|
||||
write_mode('naive')
|
||||
compose('up', '-d', '--build')
|
||||
wait_ready()
|
||||
token, chat_id = register_and_chat()
|
||||
naive_race = reads_after_write(token, chat_id)
|
||||
naive_cold = cold_reads(token, chat_id)
|
||||
print('off mode', flush=True)
|
||||
write_mode('off')
|
||||
recreate_api(build=False)
|
||||
off_repeat = warm_reads(token, chat_id)
|
||||
print('protected mode', flush=True)
|
||||
write_mode('protected')
|
||||
recreate_api(build=False)
|
||||
protected_race = reads_after_write(token, chat_id)
|
||||
protected_cold = cold_reads(token, chat_id)
|
||||
protected_miss_p50 = miss_latency(token, chat_id)
|
||||
protected_warm = warm_reads(token, chat_id)
|
||||
print('redis restart', flush=True)
|
||||
restarted = redis_restart(token, chat_id, protected_race['message_id'])
|
||||
evidence = {
|
||||
'track': 1,
|
||||
'cached_limit': 50,
|
||||
'page_ttl_seconds': 30,
|
||||
'naive_fill_hold_seconds': 0.2,
|
||||
'naive': {
|
||||
'reads_started_after_201': naive_race['reads_started_after_201'],
|
||||
'reads_missing_new_message': naive_race['reads_missing_new_message'],
|
||||
'stale_read_observed': naive_race['reads_missing_new_message'] > 0,
|
||||
'cold': naive_cold,
|
||||
},
|
||||
'off': off_repeat,
|
||||
'protected': {
|
||||
'reads_started_after_201': protected_race['reads_started_after_201'],
|
||||
'reads_missing_new_message': protected_race['reads_missing_new_message'],
|
||||
'read_after_write': protected_race['reads_missing_new_message'] == 0,
|
||||
'cold': protected_cold,
|
||||
'sequential_miss_p50_ms': protected_miss_p50,
|
||||
'warm': protected_warm,
|
||||
'after_redis_restart': restarted,
|
||||
},
|
||||
}
|
||||
found = problems_of(evidence)
|
||||
print(json.dumps(evidence, indent=2), flush=True)
|
||||
if found:
|
||||
raise SystemExit('track1 failed: ' + '; '.join(found))
|
||||
EVIDENCE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
EVIDENCE_PATH.write_text(json.dumps(evidence, indent=2) + '\n', encoding='utf-8')
|
||||
print('wrote evidence/track1.json', flush=True)
|
||||
except SystemExit as exc:
|
||||
failure = exc
|
||||
finally:
|
||||
ENV_PATH.write_bytes(original)
|
||||
print('restoring protected api', flush=True)
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'api')
|
||||
wait_ready()
|
||||
compose('start', 'redis')
|
||||
compose('start', 'postgres')
|
||||
if failure is not None:
|
||||
raise failure
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user