ЛР 2 - БПИ2302 - Самохвалов - на 5 #128
@@ -1,5 +1,7 @@
|
||||
.git
|
||||
.github
|
||||
docs/plans
|
||||
LAB_NOTES.md
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
+3
-9
@@ -1,11 +1,5 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Copy to .env for the local Compose stack; never commit .env.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Create secrets/postgres_password and secrets/database_url locally before Compose.
|
||||
# Valkey is internal only and caches User projections for 60 seconds; no named volume is needed.
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Create disposable local secrets
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
from secrets import token_urlsafe
|
||||
|
||||
Path('secrets').mkdir(exist_ok=True)
|
||||
password = token_urlsafe(32)
|
||||
Path('secrets/postgres_password').write_text(password)
|
||||
Path('secrets/database_url').write_text(
|
||||
f'postgresql://messenger:{password}@postgres:5432/messenger'
|
||||
)
|
||||
Path('.env').write_text(Path('.env.example').read_text())
|
||||
PY
|
||||
- name: Build and start
|
||||
run: docker compose --env-file .env up -d --build --wait
|
||||
- name: Public API smoke
|
||||
run: make test
|
||||
- name: Persistence through full restart
|
||||
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
- name: Stop without deleting data
|
||||
if: always()
|
||||
run: docker compose --env-file .env down
|
||||
@@ -1,4 +1,6 @@
|
||||
# Local/private state
|
||||
/docs/plans/
|
||||
/LAB_NOTES.md
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
FROM python:3.12.12-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app
|
||||
COPY --chown=app:app src/ src/
|
||||
COPY --chown=app:app migrations/ migrations/
|
||||
COPY --chown=app:app scripts/migrate.py scripts/migrate.py
|
||||
COPY --chown=app:app student-tests/test_restore.py student-tests/test_restore.py
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
CMD ["sh", "-c", "python scripts/migrate.py && exec python -m src.server"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence
|
||||
.PHONY: help check validate test load test-persistence test-level4
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -24,3 +24,9 @@ load:
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-level4:
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_database.py
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_auth.py
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_cache_failures.py
|
||||
CHECK_COMPOSE_FAILURES=1 $(PYTHON) student-tests/test_compose_failures.py
|
||||
|
||||
@@ -1,37 +1,105 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Самохвалов Андрей Ремович, БПИ-2302.
|
||||
- Целевая оценка: 5, трек 2 (сравнение запросов и восстановление). Уровни 3 и 4 и выбранный трек 5 проверены на локальном стенде.
|
||||
- Ветка: `solution/bpi2302-samohvalov`; commit/PR указываются после публикации.
|
||||
- Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/openapi.json`, публичные тесты и примеры этого репозитория; FastAPI, psycopg, PostgreSQL и Argon2id. Код и инструкции подготовлены с помощью ИИ Codex и проверяются запуском команд ниже.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose, Python 3.9+ и `make`. Стенд рассчитан на учебный localhost. API собирается из исходников в `python:3.12.12-slim-bookworm`; БД — `postgres:17.6-bookworm`, кэш — `valkey/valkey:8.0.2-alpine`. Python-зависимости закреплены в `requirements.txt`; кэш использует короткий RESP-клиент стандартной библиотеки. Порт API опубликован только на `127.0.0.1:8080`; PostgreSQL и Valkey наружу не публикуются. На моей macOS с Colima команды `make test` и `make test-persistence` выполнялись на хосте с разрешённым доступом к Docker socket и loopback. В ограниченном sandbox `127.0.0.1:8080` был недоступен; проверка не запускалась внутри VM и SSH-туннель не требовался.
|
||||
|
||||
После чистого clone создать локальные файлы, которые исключены из Git:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
mkdir -p secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
from secrets import token_urlsafe
|
||||
|
||||
password = token_urlsafe(32)
|
||||
Path('secrets/postgres_password').write_text(password)
|
||||
Path('secrets/database_url').write_text(
|
||||
f'postgresql://messenger:{password}@postgres:5432/messenger'
|
||||
)
|
||||
PY
|
||||
chmod 600 .env secrets/*
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make check
|
||||
make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-level4
|
||||
docker compose --env-file .env exec -T api python - < scripts/track2_experiment.py > evidence/query-comparison.csv 2> evidence/query-plans.txt
|
||||
bash scripts/backup_restore.sh > evidence/restore-check.txt
|
||||
```
|
||||
|
||||
На каждом старте API выполняет `scripts/migrate.py` под advisory lock: миграции `001` и `002` применяются только один раз и завершаются до приёма запросов. `docker compose --env-file .env exec -T api python scripts/migrate.py` безопасно повторить вручную. `GET /health/ready` требует доступную БД. `docker compose --env-file .env down` останавливает стенд с сохранением именованного volume. Только для намеренного уничтожения учебных данных: `docker compose --env-file .env down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент → FastAPI (`api`) → PostgreSQL (`postgres`) и необязательный Valkey (`valkey`); `postgres_data` сохраняет состояние между перезапусками. PostgreSQL — источник истины для users, chats, memberships, messages и sessions. Создание чата с участниками выполняется одной транзакцией. Пароль хранится как Argon2id hash, непрозрачный токен выдаётся только клиенту, в БД лежит его SHA-256 и абсолютный срок действия. Из lab1 сохранены чаты, членство, сообщения и keyset-пагинация; `X-User-Id` заменён входом по Basic и запросами по Bearer. `.env.example` задаёт опубликованный `APP_PORT` и `SESSION_TTL_SECONDS`; `DATABASE_URL_FILE`, `VALKEY_URL`, `APP_HOST` и порт внутри контейнера установлены Compose. Pool ограничен восемью соединениями и SQL timeout 5 секунд.
|
||||
|
||||
Каждый Bearer-запрос проверяет hash, срок и отзыв сессии в PostgreSQL. После этого публичная проекция User читается по UUID из Valkey с TTL 60 секунд; на промахе или отказе кэша читается PostgreSQL. После регистрации создаётся новый UUID; endpoint изменения/удаления пользователя в lab2 нет, поэтому инвалидация при записи не требуется. Потеря Valkey очищает лишь ускоритель, не сессии или подтверждённые данные. У кэша нет именованного volume, и приложение не полагается на сохранность его данных. При потере PostgreSQL авторизация не использует даже заполненный кэш и возвращает 503; `/health/ready` зависит от БД, но не от кэша.
|
||||
|
||||
| Роль | Образ/версия | Проверка | Хранилище/секрет |
|
||||
| --- | --- | --- | --- |
|
||||
| API | Собственный build на Python 3.12.12 | `/health/ready` | Без volume; `database_url` через Docker secret |
|
||||
| PostgreSQL | postgres:17.6-bookworm | `pg_isready` | `postgres_data`; `postgres_password` через Docker secret |
|
||||
| Valkey | valkey/valkey:8.0.2-alpine | `valkey-cli ping` | Временный кэш User; без именованного volume и внешнего порта |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke | `make test BASE_URL=http://127.0.0.1:8080` | PASS, lab2 public API scenario | `evidence/restart-2026-10-02.txt` |
|
||||
| Стенд и миграции уровня 3 | `docker compose --env-file .env up -d --build --wait`; повторный `scripts/migrate.py` | API и БД healthy, повторная миграция завершилась без ошибки | `evidence/restart-2026-10-02.txt` |
|
||||
| Сохранность уровня 3 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: прежние данные и сессия после рестарта обоих сервисов | `evidence/restart-2026-10-02.txt` |
|
||||
| Отказ обязательной БД | Остановить `postgres`, запросить readiness и защищённый endpoint | Оба HTTP 503; затем БД восстановлена, сервисы healthy | `evidence/restart-2026-10-02.txt` |
|
||||
| Кэш и отказы уровня 4 | `docker compose exec -T api python - < student-tests/test_cache_failures.py`; отказ Valkey/БД | Два интеграционных теста PASS; SQL 2/1/2 на miss/hit/reload; Valkey hit/miss 1/2; smoke проходит без кэша; при отказе БД 503, отозванный токен после рестарта кэша 401 | `evidence/cache-2026-10-02.txt` |
|
||||
| Негативные, конкурентные проверки уровня 4 | `make test-level4` | 10 тестов PASS: короткий TTL 2 с и неизменный expiry после рестарта API, logout/повторный logout, одинаковая ошибка неверных credentials, гонка username 201/409, откат чата после ошибки второй membership, обход отказавшего Valkey, ограниченный 503 при отказе БД и 401 после перезапуска Valkey. Оба списка отвечают 400 на limit из 4301 цифры | `evidence/level4-tests.txt` |
|
||||
| Индекс истории уровня 4 | `docker compose --env-file .env exec -T api python - --count 10000 --deep-offset 8000 < scripts/seed_history.py` | 10 000 сообщений в одном чате; первая и глубокая страницы по 51 строке, порядок и отсутствие повторов проверены. Обе страницы используют `messages_history_idx` | `evidence/history-explain.txt` |
|
||||
| Трек 5: ≥100 000 сообщений и два индекса | `scripts/track2_experiment.py` в API-контейнере | Чаты 80 000/15 000/5 000; по пять измерений первой и глубокой страниц и вставки 500 строк; индекс приложения восстановлен после опыта | `evidence/query-comparison.csv`, `evidence/query-plans.txt` |
|
||||
| Трек 5: backup/restore | `bash scripts/backup_restore.sh` | Дамп 3 441 039 байт восстановлен в новый volume; все пять таблиц совпали по count и SHA-256 строк; при остановленной исходной БД восстановленный API вернул 200 с действующим токеном и 401 с истёкшим | `evidence/restore-check.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
У `api`, `postgres` и `valkey` изменились времена `StartedAt` при сохранении именованного volume; точные значения приведены в evidence. Это подтверждает фактический перезапуск трёх контейнеров. Проверка `make validate` локально пока не завершена: DNS до `files.pythonhosted.org` не позволил установить `openapi-spec-validator==0.7.2`; отдельный CI workflow устанавливает зависимости из `requirements-dev.txt` для проверки контракта.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Стенд — Colima на arm64 macOS с Docker/Compose, 2 CPU и 2 054 840 320 байт памяти. `make test-level4` запускает тесты БД, auth, кэша внутри API-контейнера и отдельный HTTP-сценарий, который последовательно останавливает/восстанавливает Valkey и PostgreSQL. Он работает только на учебном стенде с локальным `.env`, не удаляет volumes и не публикует токены. Отказ второй вставки участника инъецируется после записи чата и первой membership: отсутствие нового чата и membership после исключения доказывает rollback, в отличие от одной предварительной проверки входных данных.
|
||||
|
||||
Для SQL-плана `scripts/seed_history.py` создаёт отдельный синтетический чат с 10 000 сообщениями, выполняет `ANALYZE messages` и проверяет 51 упорядоченную строку на первой и странице после курсора 8000-й записи. Команда и полный `EXPLAIN (ANALYZE, BUFFERS)` сохранены в `evidence/history-explain.txt`. PostgreSQL 17.6 выбрал `Index Scan using messages_history_idx` с условием `chat_id` для первой страницы и `(chat_id, (created_at,id) > cursor)` для глубокой. Обе прочли лишь 3 и 4 буфера из cache, выполнились за 0,010 и 0,014 мс в этом прогоне; это время SQL на прогретых страницах, не HTTP latency и не прогноз для других размеров истории. В запросах приложения также используется keyset `(created_at,id)` с индексом `(chat_id,created_at,id)`, поэтому глубокая страница не пропускает заново первые 8000 строк.
|
||||
|
||||
### Трек 5: план запроса и стоимость записи
|
||||
|
||||
**Гипотеза.** Покрывающий индекс `(chat_id,created_at,id) INCLUDE (sender_id,text)` уберёт обращения к heap на прогретых страницах истории, но займёт больше места и не обязан дать выигрыш по времени при короткой странице. Сохраняемый индекс приложения — `(chat_id,created_at,id)`; эксперимент временно заменяет его на покрывающий и в `finally` восстанавливает исходный.
|
||||
|
||||
**Метод.** На исходном commit `e50ba0f` и Colima-стенде (arm64, 2 CPU, 2 054 840 320 байт RAM; PostgreSQL 17.6) скрипт создаёт три синтетических чата 80 000/15 000/5 000 сообщений с равномерным временем внутри каждого чата. `VACUUM (ANALYZE)` выполнен перед каждым вариантом. Для каждого индекса одна и та же проекция из пяти полей читается первой страницей и по keyset-курсору после 72 000-й записи горячего чата; размер ответа — 51 строка. Три прогрева и пять повторов `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` дают время SQL, план и cache buffers. Отдельно пять раз вставляются и подтверждаются по 500 синтетических строк в четвёртый чат под каждым индексом; измерено время клиента на `INSERT ... SELECT` вместе с COMMIT. Итоговая БД содержит ещё 5 000 строк этого теста, поэтому фактический размер при backup — 115 026 сообщений с учётом данных прежних проверок. Значения CSV содержат каждый повтор; запросы и DDL — в `scripts/track2_experiment.py`.
|
||||
|
||||
| Вариант | Первая страница: медиана SQL, shared hit | Глубокая: медиана SQL, shared hit | INSERT 500 + COMMIT: медиана / диапазон | Размер индекса после опыта |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| A: обычный btree | 0,009 мс, 6 | 0,011 мс, 6 | 2,632 мс / 2,467–3,997 | 6 799 360 байт |
|
||||
| B: covering btree | 0,011 мс, 5 | 0,010 мс, 4 | 2,638 мс / 2,382–12,557 | 11 935 744 байт |
|
||||
|
||||
Вариант B дал `Index Only Scan` и ноль `Heap Fetches` в сохранённых планах; A дал `Index Scan`. Во всех повторениях `shared_read=0`: измерялся тёплый cache. Разница медиан SQL в пределах тысячных долей миллисекунды и не доказывает ускорение B. Медленная вставка B в одном повторе показывает нестабильность, но пяти повторов недостаточно, чтобы приписать её именно индексу. Размер B больше примерно на 76%; поэтому оставлен исходный A. Сравнение последовательное A→B, записи второго варианта идут после записей первого, и опыт не включает параллельную нагрузку, холодный cache, HTTP и долговременную запись WAL. Команда из чистого clone требует поднятого Compose и локальных `.env`/secrets по разделу запуска; эксперимент добавляет синтетические строки только в учебную БД.
|
||||
|
||||
### Трек 5: восстановление
|
||||
|
||||
`scripts/backup_restore.sh` создаёт private `pg_dump -Fc` в системном временном каталоге, поднимает второй `postgres:17.6-bookworm` с **новым именованным volume**, применяет `pg_restore --exit-on-error`, запускает отдельный API с URL второй БД, затем останавливает исходный PostgreSQL на время проверки. Он не копирует исходный volume. Источник после проверки снова запущен; временные контейнеры и файлы с URL/токенами удалены, успешный новый volume оставлен для проверки. Сценарий не объявляет успех, если dump, restore, сверка или HTTP-сценарий завершились ошибкой.
|
||||
|
||||
При замере 2026-10-02 09:51 UTC новый volume `mtusi2026lab2_restore_20261002125101_21469` содержал 110 users, 26 chats, 41 memberships, 115 026 messages и 49 sessions. Для каждой таблицы совпали количество и SHA-256 потока всех строк в порядке ключа; у трёх выбранных сообщений совпали ID и SHA-256 текста. Через восстановленный API при недоступном источнике действующий токен прочитал первую страницу (200), а токен, истёкший ещё до backup, получил 401. Полный путь «старт нового PostgreSQL → restore → API ready» занял 10 с по секундным отметкам; отдельные dump/restore уложились в разрешение 1 с этой грубой шкалы. `evidence/restore-check.txt` содержит результат без токенов и raw dump.
|
||||
|
||||
Граница восстановления — согласованный снимок `pg_dump`: подтверждённые записи после момента снимка этим резервным файлом не возвращаются. Архивации WAL и непрерывного PITR здесь нет, поэтому RPO зависит от частоты backup, а измеренные 10 с не включают поиск и передачу файла с другой машины. Тест выполнялся без внешних писателей между подготовкой fingerprint и dump. Для реальной эксплуатации потребуются расписание backup и регулярная проверка restore.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Полный restart API и PostgreSQL | Тот же действующий токен читает сообщение | Публичный persistence PASS | Подтверждённая запись сохранена; время восстановления в пределах лимита hook |
|
||||
| Остановка PostgreSQL | Readiness и защищённый запрос дают ограниченный отказ | Оба HTTP 503; после запуска БД healthy | Без принятия анонимного запроса |
|
||||
| Остановка Valkey | Запросы идут через PostgreSQL | Публичный smoke PASS при остановленном кэше; затем Valkey healthy | Данные и сессии не зависят от кэша |
|
||||
| Остановка PostgreSQL при заполненном User-кэше | Кэш не авторизует без БД | Readiness и защищённый запрос 503; после восстановления действующий токен 200 | HTTP-клиенту нужен timeout больше 5-секундного ожидания pool |
|
||||
| Отзыв токена и рестарт Valkey | Старый токен остаётся недействительным | После restart кэша прежний токен 401 | Отзыв хранится в PostgreSQL |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Подтверждён рестарт одного API, одного PostgreSQL и Valkey с сохранением volume БД и отдельное восстановление во второй PostgreSQL. Это не высокая доступность: отказ рабочей БД временно делает API недоступным, а заполненный кэш не может заменить проверку сессии. Локальный HTTP не предназначен для удалённой передачи паролей без TLS. Результаты трека 5 описывают один локальный стенд и тёплый cache; для выбора индекса под иным распределением данных опыт надо повторить.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
postgres_restore:
|
||||
image: postgres:17.6-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- restore_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
|
||||
api_restore:
|
||||
build: .
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url_restore
|
||||
VALKEY_URL: redis://valkey:6379/0
|
||||
secrets:
|
||||
- database_url_restore
|
||||
depends_on:
|
||||
postgres_restore:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
|
||||
secrets:
|
||||
database_url_restore:
|
||||
file: ${RESTORE_DATABASE_URL_FILE:?Set by scripts/backup_restore.sh}
|
||||
|
||||
volumes:
|
||||
restore_data:
|
||||
name: ${RESTORE_VOLUME_NAME:?Set by scripts/backup_restore.sh}
|
||||
@@ -0,0 +1,57 @@
|
||||
name: mtusi2026lab2
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
|
||||
api:
|
||||
build: .
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
VALKEY_URL: redis://valkey:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.2-alpine
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,48 @@
|
||||
Lab 2 U4: Valkey User cache, 2026-10-02, macOS/Colima arm64.
|
||||
Commands were run on the local Compose stack with PostgreSQL data volume preserved.
|
||||
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
PASS: api, postgres, valkey healthy; valkey/valkey:8.0.2-alpine.
|
||||
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_cache_failures.py
|
||||
Ran 2 tests in 0.207s: OK.
|
||||
SQL queries (miss/hit/reload): 2/1/2; Valkey hit/miss deltas: 1/2.
|
||||
The reload deletes only the disposable test user's cache key.
|
||||
The SQL counter wraps real psycopg connection.execute calls in the Store process.
|
||||
Every authenticated request still checks its session in PostgreSQL; only User lookup is cached.
|
||||
The test also points Valkey at a closed TCP port and verifies PostgreSQL fallback,
|
||||
then revokes the token and verifies 401.
|
||||
|
||||
make test BASE_URL=http://127.0.0.1:8080
|
||||
PASS: lab2 public API scenario with Valkey healthy.
|
||||
|
||||
docker compose --env-file .env stop valkey
|
||||
make test BASE_URL=http://127.0.0.1:8080
|
||||
docker compose --env-file .env up -d --wait valkey
|
||||
PASS: same public API scenario with Valkey stopped; it was restored healthy.
|
||||
|
||||
Before persistence hook, docker inspect StartedAt (UTC):
|
||||
api 2026-10-02T08:05:51.833412908Z
|
||||
postgres 2026-10-02T07:24:25.230933733Z
|
||||
valkey 2026-10-02T08:05:51.322012488Z
|
||||
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
|
||||
PASS: persistence preserved confirmed data and existing session.
|
||||
|
||||
After persistence hook, docker inspect StartedAt (UTC):
|
||||
api 2026-10-02T08:07:11.335901585Z
|
||||
postgres 2026-10-02T08:07:08.754444696Z
|
||||
valkey 2026-10-02T08:07:08.754742886Z
|
||||
All three containers really restarted; PostgreSQL volume was not deleted.
|
||||
|
||||
Manual HTTP failure run (one token held in process memory, no token printed):
|
||||
Warm User cache through two authenticated GET /users/me requests.
|
||||
Stop PostgreSQL: /health/ready=503, authenticated cached-User request=503.
|
||||
Restore PostgreSQL: authenticated request=200.
|
||||
Revoke session: DELETE /auth/sessions/current=204.
|
||||
Stop/start Valkey: same revoked token still receives 401.
|
||||
The HTTP observer needed timeout=15s; timeout=5s expired at the same boundary
|
||||
as the configured PostgreSQL pool timeout and did not observe the 503.
|
||||
|
||||
make check
|
||||
PASS: skeleton structure, refs and Python syntax (not an application test).
|
||||
@@ -0,0 +1,28 @@
|
||||
Lab 2 U5 history index evidence
|
||||
Base commit at measurement: d59d3f4; local Colima Docker: 2 CPUs, 2054840320 bytes memory.
|
||||
Command: docker compose --env-file .env exec -T api python - --count 10000 --deep-offset 8000 < scripts/seed_history.py
|
||||
Each run creates a new synthetic user/chat and appends its own messages to the disposable test database.
|
||||
UTC: 2026-10-02T08:41:04.204691+00:00
|
||||
PostgreSQL: 17.6 (Debian 17.6-2.pgdg12+1)
|
||||
Synthetic chat messages: 10000
|
||||
Deep cursor ordinal: 8000
|
||||
History index: CREATE INDEX messages_history_idx ON public.messages USING btree (chat_id, created_at, id)
|
||||
Verified: first/deep pages each have 51 ordered rows with no shared IDs
|
||||
|
||||
First page: EXPLAIN (ANALYZE, BUFFERS)
|
||||
Limit (cost=0.29..6.76 rows=51 width=70) (actual time=0.003..0.007 rows=51 loops=1)
|
||||
Buffers: shared hit=3
|
||||
-> Index Scan using messages_history_idx on messages (cost=0.29..1269.95 rows=10000 width=70) (actual time=0.003..0.005 rows=51 loops=1)
|
||||
Index Cond: (chat_id = '86684ee0-6818-4f81-ad7f-e7bf9069b8e8'::uuid)
|
||||
Buffers: shared hit=3
|
||||
Planning Time: 0.014 ms
|
||||
Execution Time: 0.010 ms
|
||||
|
||||
Deep page: EXPLAIN (ANALYZE, BUFFERS)
|
||||
Limit (cost=0.29..11.51 rows=51 width=70) (actual time=0.003..0.010 rows=51 loops=1)
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using messages_history_idx on messages (cost=0.29..439.78 rows=1996 width=70) (actual time=0.003..0.007 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '86684ee0-6818-4f81-ad7f-e7bf9069b8e8'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 08:41:04.152402+00'::timestamp with time zone, '492d2f70-1f2b-4e98-a87d-5bab4668d05c'::uuid)))
|
||||
Buffers: shared hit=4
|
||||
Planning Time: 0.016 ms
|
||||
Execution Time: 0.014 ms
|
||||
@@ -0,0 +1,30 @@
|
||||
Lab 2 U5 level-4 checks, 2026-10-02, base commit d59d3f4 plus local changes.
|
||||
Local Colima Docker: 2 CPUs, 2054840320 bytes memory; disposable Compose stack.
|
||||
Command: make test-level4 (secrets and session tokens never printed).
|
||||
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_database.py
|
||||
...
|
||||
----------------------------------------------------------------------
|
||||
Ran 3 tests in 1.252s
|
||||
|
||||
OK
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_auth.py
|
||||
....
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 tests in 5.335s
|
||||
|
||||
OK
|
||||
docker compose --env-file .env exec -T api python - < student-tests/test_cache_failures.py
|
||||
..
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 0.300s
|
||||
|
||||
OK
|
||||
SQL queries (miss/hit/reload): 2/1/2; Valkey hit/miss deltas: 1/2
|
||||
CHECK_COMPOSE_FAILURES=1 python3 student-tests/test_compose_failures.py
|
||||
.
|
||||
----------------------------------------------------------------------
|
||||
Ran 1 test in 13.472s
|
||||
|
||||
OK
|
||||
Valkey outage: 200; PostgreSQL outage: readiness/auth 503; revoked token after Valkey restart: 401
|
||||
@@ -0,0 +1,31 @@
|
||||
variant,index,operation,trial,dataset_rows,execution_ms,shared_hit,shared_read,shared_dirtied,plan,rows_inserted
|
||||
A,messages_history_idx,first,1,100000,0.008,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,first,2,100000,0.015,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,first,3,100000,0.011,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,first,4,100000,0.009,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,first,5,100000,0.009,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,deep,1,100000,0.009,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,deep,2,100000,0.017,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,deep,3,100000,0.011,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,deep,4,100000,0.022,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,deep,5,100000,0.009,6,0,0,Limit > messages_history_idx,0
|
||||
A,messages_history_idx,write500,1,100000,3.997,,,,INSERT 500 + COMMIT,500
|
||||
A,messages_history_idx,write500,2,100000,3.276,,,,INSERT 500 + COMMIT,500
|
||||
A,messages_history_idx,write500,3,100000,2.616,,,,INSERT 500 + COMMIT,500
|
||||
A,messages_history_idx,write500,4,100000,2.632,,,,INSERT 500 + COMMIT,500
|
||||
A,messages_history_idx,write500,5,100000,2.467,,,,INSERT 500 + COMMIT,500
|
||||
B,messages_history_covering_idx,first,1,100000,0.009,5,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,first,2,100000,0.011,5,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,first,3,100000,0.013,5,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,first,4,100000,0.01,5,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,first,5,100000,0.016,5,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,deep,1,100000,0.009,4,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,deep,2,100000,0.009,4,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,deep,3,100000,0.011,4,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,deep,4,100000,0.012,4,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,deep,5,100000,0.01,4,0,0,Limit > messages_history_covering_idx,0
|
||||
B,messages_history_covering_idx,write500,1,100000,2.638,,,,INSERT 500 + COMMIT,500
|
||||
B,messages_history_covering_idx,write500,2,100000,12.557,,,,INSERT 500 + COMMIT,500
|
||||
B,messages_history_covering_idx,write500,3,100000,2.7,,,,INSERT 500 + COMMIT,500
|
||||
B,messages_history_covering_idx,write500,4,100000,2.382,,,,INSERT 500 + COMMIT,500
|
||||
B,messages_history_covering_idx,write500,5,100000,2.439,,,,INSERT 500 + COMMIT,500
|
||||
|
@@ -0,0 +1,11 @@
|
||||
UTC: 2026-10-02T09:42:09.839961+00:00
|
||||
PostgreSQL: 17.6 (Debian 17.6-2.pgdg12+1)
|
||||
Synthetic chat sizes: hot=80000, medium=15000, cold=5000; fixed 51-row pages; deep cursor=72000
|
||||
Variant A: messages_history_idx
|
||||
{"variant": "A", "operation": "first", "plan": {"Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 0.42, "Total Cost": 7.65, "Plan Rows": 51, "Plan Width": 75, "Actual Startup Time": 0.001, "Actual Total Time": 0.006, "Actual Rows": 51, "Actual Loops": 1, "Shared Hit Blocks": 6, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0, "Plans": [{"Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "messages_history_idx", "Relation Name": "messages", "Alias": "messages", "Startup Cost": 0.42, "Total Cost": 11340.4, "Plan Rows": 80004, "Plan Width": 75, "Actual Startup Time": 0.001, "Actual Total Time": 0.004, "Actual Rows": 51, "Actual Loops": 1, "Index Cond": "(chat_id = 'a9024646-24db-4984-b912-1030eea3ed60'::uuid)", "Rows Removed by Index Recheck": 0, "Shared Hit Blocks": 6, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0}]}}
|
||||
{"variant": "A", "operation": "deep", "plan": {"Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 0.42, "Total Cost": 53.72, "Plan Rows": 51, "Plan Width": 75, "Actual Startup Time": 0.002, "Actual Total Time": 0.006, "Actual Rows": 51, "Actual Loops": 1, "Shared Hit Blocks": 6, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0, "Plans": [{"Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "messages_history_idx", "Relation Name": "messages", "Alias": "messages", "Startup Cost": 0.42, "Total Cost": 6225.66, "Plan Rows": 5956, "Plan Width": 75, "Actual Startup Time": 0.001, "Actual Total Time": 0.004, "Actual Rows": 51, "Actual Loops": 1, "Index Cond": "((chat_id = 'a9024646-24db-4984-b912-1030eea3ed60'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 09:42:09.355081+00'::timestamp with time zone, '3101b2d4-b2be-4696-9cf8-ea625d0e9d29'::uuid)))", "Rows Removed by Index Recheck": 0, "Shared Hit Blocks": 6, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0}]}}
|
||||
Variant B: messages_history_covering_idx
|
||||
{"variant": "B", "operation": "first", "plan": {"Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 0.42, "Total Cost": 3.89, "Plan Rows": 51, "Plan Width": 75, "Actual Startup Time": 0.002, "Actual Total Time": 0.006, "Actual Rows": 51, "Actual Loops": 1, "Shared Hit Blocks": 5, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0, "Plans": [{"Node Type": "Index Only Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "messages_history_covering_idx", "Relation Name": "messages", "Alias": "messages", "Startup Cost": 0.42, "Total Cost": 5458.29, "Plan Rows": 80107, "Plan Width": 75, "Actual Startup Time": 0.002, "Actual Total Time": 0.005, "Actual Rows": 51, "Actual Loops": 1, "Index Cond": "(chat_id = 'a9024646-24db-4984-b912-1030eea3ed60'::uuid)", "Rows Removed by Index Recheck": 0, "Heap Fetches": 0, "Shared Hit Blocks": 5, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0}]}}
|
||||
{"variant": "B", "operation": "deep", "plan": {"Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 0.42, "Total Cost": 4.04, "Plan Rows": 51, "Plan Width": 75, "Actual Startup Time": 0.002, "Actual Total Time": 0.006, "Actual Rows": 51, "Actual Loops": 1, "Shared Hit Blocks": 4, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0, "Plans": [{"Node Type": "Index Only Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "messages_history_covering_idx", "Relation Name": "messages", "Alias": "messages", "Startup Cost": 0.42, "Total Cost": 523.64, "Plan Rows": 7361, "Plan Width": 75, "Actual Startup Time": 0.002, "Actual Total Time": 0.004, "Actual Rows": 51, "Actual Loops": 1, "Index Cond": "((chat_id = 'a9024646-24db-4984-b912-1030eea3ed60'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 09:42:09.355081+00'::timestamp with time zone, '3101b2d4-b2be-4696-9cf8-ea625d0e9d29'::uuid)))", "Rows Removed by Index Recheck": 0, "Heap Fetches": 0, "Shared Hit Blocks": 4, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0}]}}
|
||||
Original history index restored; committed write rows=5000
|
||||
Index size after benchmark, original A=6799360 bytes, covering B=11935744 bytes; B dropped after measurement
|
||||
@@ -0,0 +1,30 @@
|
||||
Lab 2 local Compose verification, 2026-10-02 (UTC timestamps)
|
||||
Checks ran on the macOS host with Docker socket and loopback access enabled;
|
||||
the restricted sandbox could not reach 127.0.0.1:8080. No VM-local run or
|
||||
SSH tunnel was used.
|
||||
|
||||
docker compose --env-file .env config --quiet -> exit 0
|
||||
docker compose --env-file .env up -d --build --wait -> API and PostgreSQL healthy
|
||||
make test BASE_URL=http://127.0.0.1:8080 -> PASS: lab2 public API scenario
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
|
||||
-> PASS: persistence preserved confirmed data and existing session
|
||||
|
||||
docker inspect before persistence hook:
|
||||
api 2026-10-02T07:18:38.19716Z
|
||||
postgres 2026-10-02T07:18:37.971410919Z
|
||||
docker inspect after persistence hook:
|
||||
api 2026-10-02T07:20:21.164948401Z
|
||||
postgres 2026-10-02T07:20:18.602237039Z
|
||||
Both service StartedAt values changed; container IDs stayed constant. Compose retained
|
||||
the named mtusi2026lab2_postgres_data volume. No volume deletion or seed occurred.
|
||||
|
||||
With PostgreSQL stopped, GET /health/ready and GET /api/v1/users/me with a
|
||||
syntactically valid but unknown Bearer each returned HTTP 503. Compose then
|
||||
started PostgreSQL again; both services returned healthy.
|
||||
After recovery, make test passed again. Querying schema_migrations returned
|
||||
one row for version 1 and one row for version 2 after repeated migration runs.
|
||||
|
||||
make check -> PASS: offline skeleton structure, local refs and Python syntax
|
||||
make validate PYTHON=.venv/bin/python -> not run successfully: the pinned
|
||||
openapi-spec-validator package was absent and pip could not resolve
|
||||
files.pythonhosted.org in this environment. The CI job installs it separately.
|
||||
@@ -0,0 +1,19 @@
|
||||
UTC start: 2026-10-02T09:51:09Z
|
||||
Source: mtusi2026lab2_postgres_data; new volume: mtusi2026lab2_restore_20261002125101_21469
|
||||
pg_dump custom-format backup: 3441039 bytes, 0 seconds
|
||||
pg_restore into fresh volume: 0 seconds
|
||||
New PostgreSQL + pg_restore + restored API ready: 10 seconds
|
||||
Source PostgreSQL stopped; verification uses only postgres_restore and api_restore
|
||||
Restore fingerprint PASS (all rows, ordered by primary key):
|
||||
users: count=110 sha256=84e13a36176df82d787eeb17cf160e722adb27b354efc63cb0b071b60cb3d0dc
|
||||
chats: count=26 sha256=53597c37349eeb1dd2dcc4b844ebd0daf609ec94b0f0b761c49f2494d8d5f283
|
||||
memberships: count=41 sha256=04f4aa2eda1dac52d50043574125826e294c572f9e1ca38e9ae871ad12ac3897
|
||||
messages: count=115026 sha256=f59985688652b2c9a4217847a4c4967ab7763714913fc59980e6e905bb43aa9d
|
||||
sessions: count=49 sha256=81c9bd6d920343331e8855ffccf7122f21f02778decedd3f4cc3bb76c8f62147
|
||||
Selected synthetic messages PASS (id, text SHA-256):
|
||||
c00beec9-c124-44c5-8a91-0629359b0fbc 535c5b38862ae8ece532e63f3f4003ed180a8d072364909e203a07c0e586f564
|
||||
ebe3793f-8a8e-4ea7-b440-b4d6b98758cb 0bef8c417a079915200fb7168a9ccd09a91f72761098499885c55c762c98e125
|
||||
7d0b0c48-437f-45d0-98ab-05d80ee615a6 160bf34ba510916e90b48de2484fd9e055fdff06a103da43c94488ca43736fd0
|
||||
Restored API PASS: readiness 200, active token reads first message 200, expired token 401
|
||||
UTC verified: 2026-10-02T09:51:20Z
|
||||
Recovery point: pg_dump snapshot; writes committed afterward are outside this backup
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX chats_history_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE memberships (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload.
|
||||
-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8');
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check
|
||||
CHECK (octet_length(text) <= 16000);
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi==0.118.3
|
||||
pydantic==2.12.4
|
||||
uvicorn==0.38.0
|
||||
psycopg[binary,pool]==3.2.12
|
||||
psycopg-pool==3.3.3
|
||||
argon2-cffi==25.1.0
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
RESTORE_VOLUME_NAME="mtusi2026lab2_restore_$(date +%Y%m%d%H%M%S)_$$"
|
||||
export RESTORE_VOLUME_NAME
|
||||
work="$(mktemp -d)"
|
||||
RESTORE_DATABASE_URL_FILE="$PWD/secrets/database_url_restore_${RESTORE_VOLUME_NAME}"
|
||||
export RESTORE_DATABASE_URL_FILE
|
||||
source_stopped=0
|
||||
cleanup() {
|
||||
if [ "$source_stopped" -eq 1 ]; then
|
||||
docker compose --env-file .env start postgres >/dev/null
|
||||
fi
|
||||
docker compose --env-file .env -f compose.yaml -f compose.restore.yaml stop api_restore postgres_restore >/dev/null 2>&1 || true
|
||||
docker compose --env-file .env -f compose.yaml -f compose.restore.yaml rm -sf api_restore postgres_restore >/dev/null 2>&1 || true
|
||||
rm -f "$RESTORE_DATABASE_URL_FILE"
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Derive a private URL for the new service without printing the password.
|
||||
python3 - "$RESTORE_DATABASE_URL_FILE" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
url = urlsplit(Path('secrets/database_url').read_text().strip())
|
||||
assert url.hostname == 'postgres', 'expected source hostname postgres'
|
||||
host = url.netloc.replace('@postgres:', '@postgres_restore:')
|
||||
assert host != url.netloc
|
||||
Path(sys.argv[1]).write_text(urlunsplit(url._replace(netloc=host)))
|
||||
PY
|
||||
chmod 600 "$RESTORE_DATABASE_URL_FILE"
|
||||
|
||||
compose=(docker compose --env-file .env -f compose.yaml -f compose.restore.yaml)
|
||||
docker compose --env-file .env up -d --build --wait api >/dev/null
|
||||
docker compose --env-file .env exec -T -e PYTHONPATH=/app api python student-tests/test_restore.py prepare > "$work/expected.json"
|
||||
chmod 600 "$work/expected.json"
|
||||
echo "UTC start: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Source: mtusi2026lab2_postgres_data; new volume: $RESTORE_VOLUME_NAME"
|
||||
backup_start=$(date +%s)
|
||||
docker compose --env-file .env exec -T postgres pg_dump -Fc -U messenger -d messenger > "$work/backup.dump"
|
||||
backup_end=$(date +%s)
|
||||
echo "pg_dump custom-format backup: $(wc -c < "$work/backup.dump") bytes, $((backup_end-backup_start)) seconds"
|
||||
recovery_start=$(date +%s)
|
||||
"${compose[@]}" up -d --wait postgres_restore >/dev/null
|
||||
restore_start=$(date +%s)
|
||||
"${compose[@]}" exec -T postgres_restore pg_restore --exit-on-error -U messenger -d messenger --no-owner --no-acl < "$work/backup.dump"
|
||||
restore_end=$(date +%s)
|
||||
echo "pg_restore into fresh volume: $((restore_end-restore_start)) seconds"
|
||||
"${compose[@]}" up -d --build --wait api_restore >/dev/null
|
||||
recovery_end=$(date +%s)
|
||||
echo "New PostgreSQL + pg_restore + restored API ready: $((recovery_end-recovery_start)) seconds"
|
||||
docker compose --env-file .env stop postgres >/dev/null
|
||||
source_stopped=1
|
||||
echo "Source PostgreSQL stopped; verification uses only postgres_restore and api_restore"
|
||||
"${compose[@]}" exec -T -e PYTHONPATH=/app api_restore python student-tests/test_restore.py verify < "$work/expected.json"
|
||||
echo "UTC verified: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Recovery point: pg_dump snapshot; writes committed afterward are outside this backup"
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Apply numbered SQL migrations once, atomically and under a DB lock."""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.db import database_url
|
||||
|
||||
|
||||
def main() -> None:
|
||||
migration_dir = Path(__file__).resolve().parents[1] / "migrations"
|
||||
with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2026, 2)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
|
||||
)
|
||||
for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")):
|
||||
version = int(path.name.split("_", 1)[0])
|
||||
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
|
||||
continue
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
# Stop and start API, PostgreSQL and Valkey without deleting PostgreSQL data.
|
||||
# Compose preserves named volumes because neither command removes them.
|
||||
docker compose --env-file .env stop
|
||||
docker compose --env-file .env up -d --wait
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Seed one synthetic chat and explain the two real history page queries."""
|
||||
|
||||
import argparse
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from src.store import Store, connection
|
||||
|
||||
|
||||
def explain(conn, label, query, params):
|
||||
print(f"\n{label}: EXPLAIN (ANALYZE, BUFFERS)")
|
||||
for row in conn.execute("EXPLAIN (ANALYZE, BUFFERS) " + query, params):
|
||||
print(row["QUERY PLAN"])
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--count", type=int, default=10_000)
|
||||
parser.add_argument("--deep-offset", type=int, default=8_000)
|
||||
args = parser.parse_args()
|
||||
if args.count < 10_000 or not 0 < args.deep_offset < args.count - 50:
|
||||
parser.error("count must be >=10000 and deep-offset must leave a full 50-row page")
|
||||
|
||||
store = Store()
|
||||
user = store.register("u_" + uuid4().hex[:16], "Synthetic history", secrets.token_urlsafe(18))
|
||||
chat = store.create_chat(user["id"], "Synthetic history", [user["id"]])
|
||||
base = datetime.now(timezone.utc)
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT gen_random_uuid(), %s, %s, convert_to('synthetic-' || n, 'UTF8'), "
|
||||
"%s::timestamptz + n * interval '1 microsecond' "
|
||||
"FROM generate_series(1,%s) AS n",
|
||||
(chat["id"], user["id"], base, args.count),
|
||||
)
|
||||
conn.execute("ANALYZE messages")
|
||||
count = conn.execute("SELECT count(*) FROM messages WHERE chat_id=%s", (chat["id"],)).fetchone()["count"]
|
||||
assert count == args.count
|
||||
index = conn.execute(
|
||||
"SELECT indexdef FROM pg_indexes WHERE tablename='messages' AND indexname='messages_history_idx'"
|
||||
).fetchone()["indexdef"]
|
||||
cursor = conn.execute(
|
||||
"SELECT created_at,id FROM messages WHERE chat_id=%s "
|
||||
"ORDER BY created_at,id OFFSET %s LIMIT 1",
|
||||
(chat["id"], args.deep_offset - 1),
|
||||
).fetchone()
|
||||
projection = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"
|
||||
first = projection + " ORDER BY created_at,id LIMIT 51"
|
||||
deep = projection + " AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51"
|
||||
first_params = (chat["id"],)
|
||||
deep_params = (chat["id"], cursor["created_at"], cursor["id"])
|
||||
first_rows = conn.execute(first, first_params).fetchall()
|
||||
deep_rows = conn.execute(deep, deep_params).fetchall()
|
||||
assert len(first_rows) == len(deep_rows) == 51
|
||||
assert not {row["id"] for row in first_rows} & {row["id"] for row in deep_rows}
|
||||
assert all((a["created_at"], a["id"]) < (b["created_at"], b["id"])
|
||||
for rows in (first_rows, deep_rows) for a, b in zip(rows, rows[1:]))
|
||||
print("Lab 2 U5 history index evidence")
|
||||
print("UTC:", datetime.now(timezone.utc).isoformat())
|
||||
print("PostgreSQL:", conn.execute("SHOW server_version").fetchone()["server_version"])
|
||||
print("Synthetic chat messages:", count)
|
||||
print("Deep cursor ordinal:", args.deep_offset)
|
||||
print("History index:", index)
|
||||
print("Verified: first/deep pages each have 51 ordered rows with no shared IDs")
|
||||
explain(conn, "First page", first, first_params)
|
||||
explain(conn, "Deep page", deep, deep_params)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Reproducible 100k-row history-index and insert-cost experiment."""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import secrets
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from time import perf_counter
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from src.db import database_url
|
||||
from src.store import Store
|
||||
|
||||
|
||||
HISTORY = "CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id)"
|
||||
COVERING = (
|
||||
"CREATE INDEX messages_history_covering_idx ON messages "
|
||||
"(chat_id, created_at, id) INCLUDE (sender_id, text)"
|
||||
)
|
||||
PROJECTION = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"
|
||||
|
||||
|
||||
def explain(conn, query, params):
|
||||
result = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query, params).fetchone()["QUERY PLAN"][0]
|
||||
plan = result["Plan"]
|
||||
names = []
|
||||
|
||||
def visit(node):
|
||||
names.append(node.get("Index Name", node["Node Type"]))
|
||||
for child in node.get("Plans", []):
|
||||
visit(child)
|
||||
|
||||
visit(plan)
|
||||
return result, " > ".join(names)
|
||||
|
||||
|
||||
def main():
|
||||
store = Store()
|
||||
user = store.register("track2_" + uuid4().hex[:16], "Synthetic benchmark", secrets.token_urlsafe(18))
|
||||
sizes = (("hot", 80_000), ("medium", 15_000), ("cold", 5_000))
|
||||
chats = [(name, size, store.create_chat(user["id"], "Track 2 " + name, [user["id"]])["id"])
|
||||
for name, size in sizes]
|
||||
write_chat = store.create_chat(user["id"], "Track 2 writes", [user["id"]])["id"]
|
||||
with psycopg.connect(database_url(), autocommit=True, row_factory=dict_row) as conn:
|
||||
conn.execute("SET statement_timeout=0")
|
||||
base = datetime.now(timezone.utc)
|
||||
for name, size, chat_id in chats:
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT gen_random_uuid(),%s,%s,convert_to(%s || n, 'UTF8'),%s::timestamptz + n * interval '1 microsecond' "
|
||||
"FROM generate_series(1,%s) n",
|
||||
(chat_id, user["id"], name + "-synthetic-", base, size),
|
||||
)
|
||||
conn.execute("VACUUM (ANALYZE) messages")
|
||||
total = sum(conn.execute("SELECT count(*) AS n FROM messages WHERE chat_id=%s", (chat_id,))
|
||||
.fetchone()["n"] for _, _, chat_id in chats)
|
||||
assert total == 100_000
|
||||
hot_id = chats[0][2]
|
||||
cursor = conn.execute(
|
||||
"SELECT created_at,id FROM messages WHERE chat_id=%s "
|
||||
"ORDER BY created_at,id OFFSET 71999 LIMIT 1", (hot_id,)
|
||||
).fetchone()
|
||||
queries = (
|
||||
("first", PROJECTION + " ORDER BY created_at,id LIMIT 51", (hot_id,)),
|
||||
("deep", PROJECTION + " AND (created_at,id)>(%s,%s) "
|
||||
"ORDER BY created_at,id LIMIT 51", (hot_id, cursor["created_at"], cursor["id"])),
|
||||
)
|
||||
writer = csv.writer(sys.stdout, lineterminator="\n")
|
||||
writer.writerow(("variant", "index", "operation", "trial", "dataset_rows", "execution_ms",
|
||||
"shared_hit", "shared_read", "shared_dirtied", "plan", "rows_inserted"))
|
||||
print(f"UTC: {datetime.now(timezone.utc).isoformat()}", file=sys.stderr)
|
||||
print(f"PostgreSQL: {conn.execute('SHOW server_version').fetchone()['server_version']}", file=sys.stderr)
|
||||
print("Synthetic chat sizes: hot=80000, medium=15000, cold=5000; fixed 51-row pages; deep cursor=72000", file=sys.stderr)
|
||||
try:
|
||||
conn.execute("DROP INDEX IF EXISTS messages_history_covering_idx")
|
||||
conn.execute(HISTORY.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1))
|
||||
for variant, index in (("A", "messages_history_idx"),
|
||||
("B", "messages_history_covering_idx")):
|
||||
if variant == "B":
|
||||
conn.execute("DROP INDEX messages_history_idx")
|
||||
conn.execute(COVERING)
|
||||
conn.execute("VACUUM (ANALYZE) messages")
|
||||
print(f"Variant {variant}: {index}", file=sys.stderr)
|
||||
for label, query, params in queries:
|
||||
expected = [row["id"] for row in conn.execute(query, params).fetchall()]
|
||||
assert len(expected) == 51
|
||||
for _ in range(3):
|
||||
explain(conn, query, params)
|
||||
for trial in range(1, 6):
|
||||
result, plan = explain(conn, query, params)
|
||||
actual = [row["id"] for row in conn.execute(query, params).fetchall()]
|
||||
assert actual == expected
|
||||
node = result["Plan"]
|
||||
writer.writerow((variant, index, label, trial, total, result["Execution Time"],
|
||||
node.get("Shared Hit Blocks", 0), node.get("Shared Read Blocks", 0),
|
||||
node.get("Shared Dirtied Blocks", 0), plan, 0))
|
||||
if trial == 1:
|
||||
print(json.dumps({"variant": variant, "operation": label, "plan": node},
|
||||
default=str), file=sys.stderr)
|
||||
# Same 500-row write statement on a separate synthetic chat under each index.
|
||||
# Five committed repetitions include index maintenance and commit time.
|
||||
for trial in range(1, 6):
|
||||
start = perf_counter()
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT gen_random_uuid(),%s,%s,convert_to('write-synthetic-' || n, 'UTF8'),"
|
||||
"clock_timestamp() + n * interval '1 microsecond' "
|
||||
"FROM generate_series(1,500) n",
|
||||
(write_chat, user["id"]),
|
||||
)
|
||||
elapsed = (perf_counter() - start) * 1000
|
||||
writer.writerow((variant, index, "write500", trial, total, round(elapsed, 3),
|
||||
"", "", "", "INSERT 500 + COMMIT", 500))
|
||||
finally:
|
||||
conn.execute("DROP INDEX IF EXISTS messages_history_covering_idx")
|
||||
conn.execute(HISTORY.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1))
|
||||
assert conn.execute(
|
||||
"SELECT count(*) AS n FROM messages WHERE chat_id=%s", (write_chat,)
|
||||
).fetchone()["n"] == 5000
|
||||
print("Original history index restored; committed write rows=5000", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
"""Messenger HTTP routes."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||
|
||||
from .auth import basic_credentials, bearer_token
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .store import store
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/health/ready")
|
||||
def ready():
|
||||
return store.ready()
|
||||
|
||||
|
||||
@router.post("/api/v1/users", status_code=201)
|
||||
def register_user(body: CreateUser):
|
||||
return store.register(body.username, body.display_name, body.password)
|
||||
|
||||
|
||||
@router.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def create_session(request: Request, response: Response):
|
||||
if await request.body():
|
||||
raise HTTPException(400)
|
||||
username, password = basic_credentials(request.headers.get("authorization"))
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return store.create_session(username, password)
|
||||
|
||||
|
||||
def current_user(request: Request) -> dict:
|
||||
return store.authenticated(bearer_token(request.headers.get("authorization")))
|
||||
|
||||
|
||||
def page_limit(raw: str) -> int:
|
||||
digits = raw.lstrip("0")
|
||||
if not raw.isdecimal() or not digits or len(digits) > 3:
|
||||
raise HTTPException(400)
|
||||
limit = int(digits)
|
||||
if not 1 <= limit <= 100:
|
||||
raise HTTPException(400)
|
||||
return limit
|
||||
|
||||
|
||||
@router.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(request: Request):
|
||||
store.revoke_session(bearer_token(request.headers.get("authorization")))
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.get("/api/v1/users/me")
|
||||
def user_me(actor: dict = Depends(current_user)):
|
||||
return actor
|
||||
|
||||
|
||||
@router.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(body: CreateChat, actor: dict = Depends(current_user)):
|
||||
return store.create_chat(actor["id"], body.title, body.member_ids)
|
||||
|
||||
|
||||
@router.get("/api/v1/chats")
|
||||
def list_chats(
|
||||
limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user)
|
||||
):
|
||||
return store.list_chats(actor["id"], page_limit(limit), cursor)
|
||||
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: UUID, actor: dict = Depends(current_user)):
|
||||
return store.chat(actor["id"], str(chat_id))
|
||||
|
||||
|
||||
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def create_message(chat_id: UUID, body: CreateMessage, actor: dict = Depends(current_user)):
|
||||
return store.create_message(actor["id"], str(chat_id), body.text)
|
||||
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(
|
||||
chat_id: UUID,
|
||||
limit: str = Query("50"),
|
||||
cursor: str | None = Query(None, min_length=1),
|
||||
actor: dict = Depends(current_user),
|
||||
):
|
||||
return store.list_messages(actor["id"], str(chat_id), page_limit(limit), cursor)
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
"""Parse HTTP credentials without exposing them in errors or logs."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z")
|
||||
|
||||
|
||||
def unauthorized(challenge: str) -> HTTPException:
|
||||
return HTTPException(401, headers={"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def basic_credentials(header: str | None) -> tuple[str, str]:
|
||||
if not header or not header.startswith("Basic "):
|
||||
raise unauthorized(BASIC_CHALLENGE)
|
||||
try:
|
||||
raw = base64.b64decode(header[6:], validate=True).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
except (binascii.Error, UnicodeError, ValueError):
|
||||
raise unauthorized(BASIC_CHALLENGE) from None
|
||||
if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128:
|
||||
raise unauthorized(BASIC_CHALLENGE)
|
||||
return username, password
|
||||
|
||||
|
||||
def bearer_token(header: str | None) -> str:
|
||||
if not header or not header.startswith("Bearer "):
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
token = header[7:]
|
||||
if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token):
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
return token
|
||||
|
||||
|
||||
def token_hash(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Best-effort cache of immutable public User projections."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
USER_TTL_SECONDS = 60
|
||||
|
||||
|
||||
def command(*parts: str) -> str | None:
|
||||
"""Send one RESP command; every call has bounded connect and read times."""
|
||||
target = urlparse(os.environ.get("VALKEY_URL", "redis://valkey:6379/0"))
|
||||
payload = b"*" + str(len(parts)).encode() + b"\r\n"
|
||||
for part in parts:
|
||||
value = part.encode("utf-8")
|
||||
payload += b"$" + str(len(value)).encode() + b"\r\n" + value + b"\r\n"
|
||||
with socket.create_connection((target.hostname or "valkey", target.port or 6379), timeout=0.5) as sock:
|
||||
sock.settimeout(0.5)
|
||||
sock.sendall(payload)
|
||||
with sock.makefile("rb") as response:
|
||||
kind = response.read(1)
|
||||
if kind in (b"+", b":"):
|
||||
return response.readline().rstrip(b"\r\n").decode("utf-8")
|
||||
if kind == b"$":
|
||||
length = int(response.readline())
|
||||
if length < 0:
|
||||
return None
|
||||
value = response.read(length)
|
||||
if len(value) != length or response.read(2) != b"\r\n":
|
||||
raise OSError("incomplete Valkey response")
|
||||
return value.decode("utf-8")
|
||||
raise OSError("invalid Valkey response")
|
||||
|
||||
|
||||
def get_user(user_id: str) -> dict | None:
|
||||
try:
|
||||
value = command("GET", "user:" + user_id)
|
||||
except (OSError, ValueError, UnicodeError):
|
||||
return None
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def put_user(user_id: str, user: dict) -> None:
|
||||
try:
|
||||
command("SET", "user:" + user_id, json.dumps(user, ensure_ascii=False), "EX", str(USER_TTL_SECONDS))
|
||||
except (OSError, ValueError, UnicodeError):
|
||||
pass
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Bounded PostgreSQL pool shared by API storage and health checks."""
|
||||
|
||||
import os
|
||||
import atexit
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
path = os.environ.get("DATABASE_URL_FILE")
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("DATABASE_URL", "postgresql:///messenger")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def pool() -> ConnectionPool:
|
||||
result = ConnectionPool(
|
||||
database_url(), min_size=0, max_size=8, timeout=5,
|
||||
kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row},
|
||||
open=True,
|
||||
)
|
||||
atexit.register(result.close)
|
||||
return result
|
||||
|
||||
|
||||
def ready() -> bool:
|
||||
try:
|
||||
with pool().connection() as conn:
|
||||
conn.execute("SELECT 1 FROM users LIMIT 0")
|
||||
return True
|
||||
except (psycopg.Error, PoolTimeout, OSError):
|
||||
return False
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""FastAPI entry point and contract-wide response handling."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from .api import router
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
|
||||
ERRORS = {
|
||||
400: ("invalid_request", "Неверный запрос"),
|
||||
401: ("unauthorized", "Требуется действительный пользователь"),
|
||||
404: ("not_found", "Объект не найден"),
|
||||
409: ("conflict", "Конфликт данных"),
|
||||
413: ("payload_too_large", "Слишком большой запрос"),
|
||||
415: ("unsupported_media_type", "Неподдерживаемый тип данных"),
|
||||
429: ("rate_limited", "Слишком много запросов"),
|
||||
503: ("unavailable", "Сервис недоступен"),
|
||||
}
|
||||
|
||||
|
||||
def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse:
|
||||
code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка"))
|
||||
request_id = request.state.request_id
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
headers={"X-Request-Id": request_id, **(extra_headers or {})},
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_contract(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions":
|
||||
media_type, _, charset = request.headers.get("content-type", "").lower().partition(";")
|
||||
if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"):
|
||||
return error_response(request, 415)
|
||||
try:
|
||||
(await request.body()).decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return error_response(request, 400)
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400)
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException):
|
||||
return error_response(request, exc.status_code, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
return error_response(request, 500)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Request bodies from the lab 1 contract."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class StrictBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
|
||||
class CreateUser(StrictBody):
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def nonblank_name(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("blank display name")
|
||||
return value
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def nonblank_password(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("blank password")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(StrictBody):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def nonblank_title(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("blank title")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def valid_members(cls, value: list[str]) -> list[str]:
|
||||
members = [str(UUID(member_id)) for member_id in value]
|
||||
if len(set(members)) != len(members):
|
||||
raise ValueError("duplicate member")
|
||||
return members
|
||||
|
||||
|
||||
class CreateMessage(StrictBody):
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
|
||||
@field_validator("text")
|
||||
@classmethod
|
||||
def nonblank_text(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("blank text")
|
||||
return value
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Run the ASGI worker as the container's main process."""
|
||||
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(
|
||||
"src.main:app",
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
)
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
"""PostgreSQL-backed users, chats and message history."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from fastapi import HTTPException
|
||||
from psycopg.errors import UniqueViolation
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from . import cache, db
|
||||
from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized
|
||||
|
||||
password_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def timestamp(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def encode_cursor(resource: str, item: dict) -> str:
|
||||
value = [resource, item["created_at"], item["id"]]
|
||||
return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None:
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)
|
||||
value = json.loads(raw)
|
||||
if not isinstance(value, list) or len(value) != 3 or value[0] != resource:
|
||||
raise ValueError
|
||||
date_text, item_id = value[1:]
|
||||
if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str):
|
||||
raise ValueError
|
||||
date = datetime.fromisoformat(date_text[:-1] + "+00:00")
|
||||
if date.utcoffset().total_seconds() != 0:
|
||||
raise ValueError
|
||||
return date, UUID(item_id)
|
||||
except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError):
|
||||
raise HTTPException(400) from None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection():
|
||||
try:
|
||||
with db.pool().connection() as conn:
|
||||
yield conn
|
||||
except (psycopg.Error, PoolTimeout, OSError):
|
||||
raise HTTPException(503) from None
|
||||
|
||||
|
||||
def user_view(row: dict) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]), "username": row["username"],
|
||||
"display_name": row["display_name"], "created_at": timestamp(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def chat_view(row: dict) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]), "title": row["title"],
|
||||
"member_ids": [str(member_id) for member_id in row["member_ids"]],
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def message_view(row: dict) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"),
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids"""
|
||||
|
||||
|
||||
class Store:
|
||||
def ready(self) -> dict:
|
||||
if not db.ready():
|
||||
raise HTTPException(503)
|
||||
return {"status": "ok"}
|
||||
|
||||
def register(self, username: str, display_name: str, password: str) -> dict:
|
||||
password_hash = password_hasher.hash(password)
|
||||
with connection() as conn:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) "
|
||||
"RETURNING id,username,display_name,created_at",
|
||||
(uuid4(), username, display_name, password_hash),
|
||||
).fetchone()
|
||||
except UniqueViolation:
|
||||
raise HTTPException(409) from None
|
||||
return user_view(row)
|
||||
|
||||
def user(self, user_id: str | None) -> dict:
|
||||
if not user_id:
|
||||
raise HTTPException(401)
|
||||
try:
|
||||
user_uuid = UUID(user_id)
|
||||
except (ValueError, TypeError):
|
||||
raise HTTPException(401) from None
|
||||
cached = cache.get_user(str(user_uuid))
|
||||
if cached is not None:
|
||||
return cached
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise HTTPException(401)
|
||||
result = user_view(row)
|
||||
cache.put_user(str(user_uuid), result)
|
||||
return result
|
||||
|
||||
def create_session(self, username: str, password: str) -> dict:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
try:
|
||||
valid = row is not None and password_hasher.verify(row["password_hash"], password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
valid = False
|
||||
if not valid:
|
||||
raise unauthorized(BASIC_CHALLENGE)
|
||||
token = secrets.token_urlsafe(32)
|
||||
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
expiry = conn.execute(
|
||||
"INSERT INTO sessions (token_hash,user_id,expires_at) "
|
||||
"VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at",
|
||||
(token_hash(token), row["id"], ttl),
|
||||
).fetchone()["expires_at"]
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)}
|
||||
|
||||
def authenticated(self, token: str) -> dict:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()",
|
||||
(token_hash(token),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
return self.user(str(row["user_id"]))
|
||||
|
||||
def revoke_session(self, token: str) -> None:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash",
|
||||
(token_hash(token),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
|
||||
def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict:
|
||||
if actor_id not in member_ids:
|
||||
raise HTTPException(400)
|
||||
members = [UUID(member_id) for member_id in member_ids]
|
||||
with connection() as conn:
|
||||
count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"]
|
||||
if count != len(members):
|
||||
raise HTTPException(400)
|
||||
row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone()
|
||||
for position, member_id in enumerate(members):
|
||||
conn.execute(
|
||||
"INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||
(row["id"], member_id, position),
|
||||
)
|
||||
return chat_view({**row, "member_ids": members})
|
||||
|
||||
def chat(self, actor_id: str, chat_id: str) -> dict:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"SELECT {CHAT_COLUMNS} FROM chats c "
|
||||
"JOIN memberships access ON access.chat_id=c.id "
|
||||
"WHERE access.user_id=%s AND c.id=%s",
|
||||
(UUID(actor_id), UUID(chat_id)),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise HTTPException(404)
|
||||
return chat_view(row)
|
||||
|
||||
def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict:
|
||||
after = decode_cursor(cursor, f"chats:{actor_id}")
|
||||
query = (f"SELECT {CHAT_COLUMNS} FROM chats c "
|
||||
"JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s")
|
||||
params = [UUID(actor_id)]
|
||||
if after:
|
||||
query += " AND (c.created_at,c.id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
query += " ORDER BY c.created_at,c.id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
with connection() as conn:
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
items = [chat_view(row) for row in rows[:limit]]
|
||||
return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None}
|
||||
|
||||
def create_message(self, actor_id: str, chat_id: str, text: str) -> dict:
|
||||
chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id)
|
||||
with connection() as conn:
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid)
|
||||
).fetchone()
|
||||
if member is None:
|
||||
raise HTTPException(404)
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) "
|
||||
"RETURNING id,chat_id,sender_id,text,created_at",
|
||||
(uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")),
|
||||
).fetchone()
|
||||
return message_view(row)
|
||||
|
||||
def list_messages(self, actor_id: str, chat_id: str, limit: int, cursor: str | None) -> dict:
|
||||
after = decode_cursor(cursor, f"messages:{chat_id}")
|
||||
chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id)
|
||||
query = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"
|
||||
params = [chat_uuid]
|
||||
if after:
|
||||
query += " AND (created_at,id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
query += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
with connection() as conn:
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid)
|
||||
).fetchone()
|
||||
if member is None:
|
||||
raise HTTPException(404)
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
items = [message_view(row) for row in rows[:limit]]
|
||||
return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None}
|
||||
|
||||
|
||||
store = Store()
|
||||
@@ -0,0 +1,177 @@
|
||||
"""HTTP and database checks for durable Basic/Bearer sessions."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from src.db import database_url
|
||||
|
||||
|
||||
URL = database_url() if os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE") else None
|
||||
|
||||
|
||||
@unittest.skipUnless(URL, "set DATABASE_URL or DATABASE_URL_FILE to a disposable PostgreSQL database")
|
||||
class AuthTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
cls.port = sock.getsockname()[1]
|
||||
cls.start_server()
|
||||
|
||||
@classmethod
|
||||
def start_server(cls, ttl="3600"):
|
||||
env = {**os.environ, "SESSION_TTL_SECONDS": ttl}
|
||||
cls.server = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)],
|
||||
env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
for _ in range(100):
|
||||
if cls.server.poll() is not None:
|
||||
raise AssertionError("API server exited during startup")
|
||||
try:
|
||||
cls.call("GET", "/health/ready", expected=200)
|
||||
return
|
||||
except (AssertionError, urllib.error.URLError):
|
||||
time.sleep(0.05)
|
||||
raise AssertionError("API server did not become ready")
|
||||
|
||||
@classmethod
|
||||
def stop_server(cls):
|
||||
cls.server.terminate()
|
||||
try:
|
||||
cls.server.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls.server.kill()
|
||||
cls.server.wait(timeout=5)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.stop_server()
|
||||
|
||||
@classmethod
|
||||
def call(cls, method, path, *, body=None, authorization=None, user_id=None, expected=200):
|
||||
headers = {"Accept": "application/json"}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
if authorization is not None:
|
||||
headers["Authorization"] = authorization
|
||||
if user_id is not None:
|
||||
headers["X-User-Id"] = user_id
|
||||
request = urllib.request.Request(
|
||||
f"http://127.0.0.1:{cls.port}{path}", data=data, headers=headers, method=method,
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=5)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
status = response.status
|
||||
response_headers = response.headers
|
||||
data = response.read()
|
||||
assert status == expected, (method, path.split("?", 1)[0], expected, status)
|
||||
request_id = response_headers.get("X-Request-Id")
|
||||
assert request_id
|
||||
payload = json.loads(data) if data else None
|
||||
if status >= 400:
|
||||
assert payload["error"]["request_id"] == request_id
|
||||
assert "password" not in data.decode().lower()
|
||||
assert "token" not in data.decode().lower()
|
||||
return payload, response_headers
|
||||
|
||||
def user(self, password="Пароль:очень-длинный-123"):
|
||||
username = "u_" + uuid4().hex[:16]
|
||||
body = {"username": username, "display_name": "Алиса", "password": password}
|
||||
user, _ = self.call("POST", "/api/v1/users", body=body, expected=201)
|
||||
self.assertNotIn("password", user)
|
||||
return user, password
|
||||
|
||||
def login(self, username, password, expected=201):
|
||||
encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
return self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, expected=expected)
|
||||
|
||||
def test_basic_bearer_logout_and_storage(self):
|
||||
alice, password = self.user()
|
||||
bob, _ = self.user()
|
||||
session, headers = self.login(alice["username"], password)
|
||||
self.assertEqual(session["token_type"], "Bearer")
|
||||
self.assertEqual(session["user"], alice)
|
||||
self.assertGreaterEqual(len(base64.urlsafe_b64decode(session["token"] + "=" * (-len(session["token"]) % 4))), 32)
|
||||
self.assertEqual(headers.get("Cache-Control"), "no-store")
|
||||
bearer = "Bearer " + session["token"]
|
||||
me, _ = self.call("GET", "/api/v1/users/me", authorization=bearer, user_id=bob["id"])
|
||||
self.assertEqual(me, alice)
|
||||
with psycopg.connect(URL) as conn:
|
||||
row = conn.execute("SELECT token_hash,expires_at FROM sessions WHERE user_id=%s", (alice["id"],)).fetchone()
|
||||
password_hash = conn.execute("SELECT password_hash FROM users WHERE id=%s", (alice["id"],)).fetchone()[0]
|
||||
self.assertEqual(row[0], hashlib.sha256(session["token"].encode()).digest())
|
||||
self.assertNotEqual(password_hash, password)
|
||||
self.assertTrue(password_hash.startswith("$argon2id$"))
|
||||
self.call("DELETE", "/api/v1/auth/sessions/current", authorization=bearer, expected=204)
|
||||
for method, path in (("GET", "/api/v1/users/me"), ("DELETE", "/api/v1/auth/sessions/current")):
|
||||
_, headers = self.call(method, path, authorization=bearer, expected=401)
|
||||
self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"')
|
||||
|
||||
def test_challenges_and_uniform_credentials(self):
|
||||
user, password = self.user()
|
||||
failures = []
|
||||
for username, candidate in ((user["username"], "wrong-password-123"), ("missing_user_123", password)):
|
||||
payload, headers = self.login(username, candidate, expected=401)
|
||||
failures.append((payload["error"]["code"], payload["error"]["message"]))
|
||||
self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"')
|
||||
self.assertEqual(failures[0], failures[1])
|
||||
encoded = base64.b64encode(f"{user['username']}:{password}".encode()).decode()
|
||||
for authorization in (None, "Basic " + encoded, "Bearer garbage"):
|
||||
_, headers = self.call("GET", "/api/v1/users/me", authorization=authorization, user_id=user["id"], expected=401)
|
||||
self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"')
|
||||
_, headers = self.call("POST", "/api/v1/auth/sessions", authorization="Bearer garbage", expected=401)
|
||||
self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"')
|
||||
self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, body={"unexpected": True}, expected=400)
|
||||
|
||||
def test_huge_page_limit_is_bad_request(self):
|
||||
user, password = self.user()
|
||||
session, _ = self.login(user["username"], password)
|
||||
bearer = "Bearer " + session["token"]
|
||||
for path in ("/api/v1/chats", f"/api/v1/chats/{uuid4()}/messages"):
|
||||
self.call("GET", path + "?limit=" + "9" * 4301, authorization=bearer, expected=400)
|
||||
self.call("GET", "/api/v1/chats?limit=0001", authorization=bearer)
|
||||
self.call("GET", "/api/v1/chats?limit=%D9%A0", authorization=bearer, expected=400)
|
||||
|
||||
def test_z_short_ttl_survives_server_restart_without_extension(self):
|
||||
self.stop_server()
|
||||
self.start_server(ttl="2")
|
||||
try:
|
||||
user, password = self.user()
|
||||
session, _ = self.login(user["username"], password)
|
||||
bearer = "Bearer " + session["token"]
|
||||
self.call("GET", "/api/v1/users/me", authorization=bearer)
|
||||
with psycopg.connect(URL) as conn:
|
||||
before = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0]
|
||||
self.stop_server()
|
||||
self.start_server(ttl="3600")
|
||||
with psycopg.connect(URL) as conn:
|
||||
after = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0]
|
||||
self.assertEqual(before, after)
|
||||
time.sleep(2.1)
|
||||
_, headers = self.call("GET", "/api/v1/users/me", authorization=bearer, expected=401)
|
||||
self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"')
|
||||
finally:
|
||||
self.stop_server()
|
||||
self.start_server()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Real PostgreSQL and Valkey checks for the optional User cache."""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
from src import cache
|
||||
from src.store import Store, connection
|
||||
|
||||
|
||||
@unittest.skipUnless((os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE"))
|
||||
and os.environ.get("VALKEY_URL"),
|
||||
"set DATABASE_URL or DATABASE_URL_FILE and VALKEY_URL for disposable services")
|
||||
class CacheTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.store = Store()
|
||||
self.user = self.store.register("u_" + uuid4().hex[:16], "Кэш", "Test-password-123")
|
||||
self.token = self.store.create_session(self.user["username"], "Test-password-123")["token"]
|
||||
|
||||
def test_hit_miss_and_sql_counts(self):
|
||||
statements = []
|
||||
|
||||
class CountingConnection:
|
||||
def __init__(self, wrapped):
|
||||
self.wrapped = wrapped
|
||||
|
||||
def execute(self, query, params=None):
|
||||
statements.append(query)
|
||||
return self.wrapped.execute(query, params)
|
||||
|
||||
@contextmanager
|
||||
def counted_connection():
|
||||
with connection() as conn:
|
||||
yield CountingConnection(conn)
|
||||
|
||||
with patch("src.store.connection", counted_connection):
|
||||
before = cache.command("INFO", "stats")
|
||||
self.assertEqual(self.store.authenticated(self.token), self.user)
|
||||
miss_sql = len(statements)
|
||||
self.assertEqual(miss_sql, 2) # session plus User on miss
|
||||
statements.clear()
|
||||
self.assertEqual(self.store.authenticated(self.token), self.user)
|
||||
hit_sql = len(statements)
|
||||
self.assertEqual(hit_sql, 1) # session only on cache hit
|
||||
cache.command("DEL", "user:" + self.user["id"])
|
||||
statements.clear()
|
||||
self.assertEqual(self.store.authenticated(self.token), self.user)
|
||||
reload_sql = len(statements)
|
||||
self.assertEqual(reload_sql, 2) # User reload after flush
|
||||
after = cache.command("INFO", "stats")
|
||||
def stat(info, name):
|
||||
return int(next(line.split(":", 1)[1] for line in info.splitlines() if line.startswith(name + ":")))
|
||||
self.assertGreaterEqual(stat(after, "keyspace_hits") - stat(before, "keyspace_hits"), 1)
|
||||
self.assertGreaterEqual(stat(after, "keyspace_misses") - stat(before, "keyspace_misses"), 2)
|
||||
print(f"SQL queries (miss/hit/reload): {miss_sql}/{hit_sql}/{reload_sql}; "
|
||||
f"Valkey hit/miss deltas: {stat(after, 'keyspace_hits') - stat(before, 'keyspace_hits')}/"
|
||||
f"{stat(after, 'keyspace_misses') - stat(before, 'keyspace_misses')}")
|
||||
|
||||
def test_cache_failure_does_not_change_authentication(self):
|
||||
with patch.dict(os.environ, {"VALKEY_URL": "redis://127.0.0.1:1/0"}):
|
||||
self.assertEqual(self.store.authenticated(self.token), self.user)
|
||||
self.store.revoke_session(self.token)
|
||||
with self.assertRaises(Exception) as failure:
|
||||
self.store.authenticated(self.token)
|
||||
self.assertEqual(failure.exception.status_code, 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Opt-in HTTP checks that stop only disposable Compose dependencies."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get("CHECK_COMPOSE_FAILURES") == "1", "opt in on the disposable Compose stack")
|
||||
class ComposeFailureTests(unittest.TestCase):
|
||||
def test_cache_bypass_database_refusal_and_revocation(self):
|
||||
client = Client(base_url="http://127.0.0.1:8080", timeout=15)
|
||||
actor = client.user("outage")
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User") # fill User cache
|
||||
|
||||
try:
|
||||
compose("stop", "valkey")
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "valkey")
|
||||
|
||||
try:
|
||||
compose("stop", "postgres")
|
||||
client.call("GET", "/health/ready", expected=503)
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, expected=503)
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "postgres")
|
||||
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
client.call("DELETE", "/api/v1/auth/sessions/current", actor=actor, expected=204)
|
||||
try:
|
||||
compose("restart", "valkey")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "valkey")
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, expected=401)
|
||||
print("Valkey outage: 200; PostgreSQL outage: readiness/auth 503; revoked token after Valkey restart: 401")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Integration checks against a disposable PostgreSQL database."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
|
||||
from src.db import database_url
|
||||
from src.store import Store, connection
|
||||
|
||||
|
||||
URL = database_url() if os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE") else None
|
||||
|
||||
|
||||
@unittest.skipUnless(URL, "set DATABASE_URL or DATABASE_URL_FILE to a disposable PostgreSQL database")
|
||||
class DatabaseTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
for _ in range(2):
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
|
||||
cls.store = Store()
|
||||
|
||||
def user(self):
|
||||
name = "u_" + uuid4().hex[:16]
|
||||
return self.store.register(name, "Студент", "Test-password-123")
|
||||
|
||||
def test_migration_and_unique_username_race(self):
|
||||
with psycopg.connect(URL) as conn:
|
||||
versions = conn.execute("SELECT version FROM schema_migrations").fetchall()
|
||||
self.assertEqual(versions, [(1,), (2,), (9,)])
|
||||
name = "u_" + uuid4().hex[:16]
|
||||
|
||||
def register(_):
|
||||
try:
|
||||
return self.store.register(name, "Студент", "Test-password-123")
|
||||
except Exception as exc:
|
||||
return exc
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
results = list(pool.map(register, range(2)))
|
||||
self.assertEqual(sum(isinstance(result, dict) for result in results), 1)
|
||||
self.assertEqual([result.status_code for result in results if not isinstance(result, dict)], [409])
|
||||
with psycopg.connect(URL) as conn:
|
||||
self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1)
|
||||
password_hash = conn.execute("SELECT password_hash FROM users WHERE username=%s", (name,)).fetchone()[0]
|
||||
self.assertTrue(password_hash.startswith("$argon2id$"))
|
||||
self.assertNotIn("password", str(next(result for result in results if isinstance(result, dict))))
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
|
||||
with psycopg.connect(URL) as conn:
|
||||
self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1)
|
||||
|
||||
def test_chat_rollback_and_keyset_history(self):
|
||||
alice, bob, eve = self.user(), self.user(), self.user()
|
||||
with psycopg.connect(URL) as conn:
|
||||
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
|
||||
with self.assertRaises(Exception) as failure:
|
||||
self.store.create_chat(alice["id"], "invalid", [alice["id"], str(uuid4())])
|
||||
self.assertEqual(failure.exception.status_code, 400)
|
||||
with psycopg.connect(URL) as conn:
|
||||
self.assertEqual(conn.execute("SELECT count(*) FROM chats").fetchone()[0], before)
|
||||
|
||||
chat = self.store.create_chat(alice["id"], "Valid", [alice["id"], bob["id"]])
|
||||
chat2 = self.store.create_chat(alice["id"], "Second", [alice["id"]])
|
||||
chat_page = self.store.list_chats(alice["id"], 1, None)
|
||||
self.assertEqual(chat_page["items"], [chat])
|
||||
self.assertEqual(self.store.list_chats(alice["id"], 1, chat_page["next_cursor"]), {"items": [chat2], "next_cursor": None})
|
||||
first = self.store.create_message(alice["id"], chat["id"], "первое")
|
||||
second = self.store.create_message(bob["id"], chat["id"], "второе")
|
||||
page = self.store.list_messages(alice["id"], chat["id"], 1, None)
|
||||
self.assertEqual(page["items"], [first])
|
||||
self.assertEqual(self.store.list_messages(bob["id"], chat["id"], 1, page["next_cursor"]), {"items": [second], "next_cursor": None})
|
||||
self.assertEqual(Store().chat(bob["id"], chat["id"]), chat)
|
||||
with self.assertRaises(Exception) as wrong_cursor:
|
||||
self.store.list_messages(alice["id"], chat2["id"], 1, page["next_cursor"])
|
||||
self.assertEqual(wrong_cursor.exception.status_code, 400)
|
||||
with self.assertRaises(Exception) as hidden:
|
||||
self.store.chat(eve["id"], chat["id"])
|
||||
self.assertEqual(hidden.exception.status_code, 404)
|
||||
|
||||
def test_membership_failure_rolls_back_inserted_chat(self):
|
||||
alice, bob = self.user(), self.user()
|
||||
title = "rollback-" + uuid4().hex
|
||||
|
||||
class FailingConnection:
|
||||
def __init__(self, wrapped):
|
||||
self.wrapped = wrapped
|
||||
self.memberships = 0
|
||||
|
||||
def execute(self, query, params=None):
|
||||
if query.startswith("INSERT INTO memberships"):
|
||||
self.memberships += 1
|
||||
if self.memberships == 2:
|
||||
raise RuntimeError("synthetic second membership failure")
|
||||
return self.wrapped.execute(query, params)
|
||||
|
||||
@contextmanager
|
||||
def failing_connection():
|
||||
with connection() as conn:
|
||||
yield FailingConnection(conn)
|
||||
|
||||
with patch("src.store.connection", failing_connection):
|
||||
with self.assertRaisesRegex(RuntimeError, "synthetic second membership failure"):
|
||||
self.store.create_chat(alice["id"], title, [alice["id"], bob["id"]])
|
||||
with psycopg.connect(URL) as conn:
|
||||
self.assertEqual(conn.execute("SELECT count(*) FROM chats WHERE title=%s", (title,)).fetchone()[0], 0)
|
||||
self.assertEqual(conn.execute(
|
||||
"SELECT count(*) FROM memberships WHERE user_id IN (%s,%s)",
|
||||
(alice["id"], bob["id"]),
|
||||
).fetchone()[0], 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Run with DATABASE_URL pointing to a disposable PostgreSQL database.
|
||||
|
||||
Checks real HTTP writes, persisted history, retries, and migration of old rows.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LAB = json.loads((ROOT / "lab.json").read_text())["number"]
|
||||
URL = os.environ.get("DATABASE_URL")
|
||||
|
||||
|
||||
@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database")
|
||||
class MessageUnicodeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
cls.port = sock.getsockname()[1]
|
||||
cls.start_server()
|
||||
cls.addClassCleanup(cls.stop_server)
|
||||
|
||||
@classmethod
|
||||
def start_server(cls):
|
||||
cls.server = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"},
|
||||
)
|
||||
for _ in range(100):
|
||||
if cls.server.poll() is not None:
|
||||
raise AssertionError("API exited during startup")
|
||||
try:
|
||||
cls.call("GET", "/health/live")
|
||||
return
|
||||
except (AssertionError, urllib.error.URLError):
|
||||
time.sleep(0.05)
|
||||
cls.stop_server()
|
||||
raise AssertionError("API did not become ready")
|
||||
|
||||
@classmethod
|
||||
def stop_server(cls):
|
||||
cls.server.terminate()
|
||||
try:
|
||||
cls.server.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls.server.kill()
|
||||
cls.server.wait(timeout=5)
|
||||
|
||||
@classmethod
|
||||
def call(cls, method, path, body=None, headers=None, expected=200):
|
||||
request = urllib.request.Request(
|
||||
f"http://127.0.0.1:{cls.port}{path}", method=method,
|
||||
data=json.dumps(body).encode() if body is not None else None,
|
||||
headers={"Content-Type": "application/json", **(headers or {})},
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=10)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
payload = response.read()
|
||||
assert response.status == expected, (method, path, response.status, expected, payload)
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
def test_http_round_trip_after_restart_and_replay(self):
|
||||
username, password = "u_" + uuid4().hex[:16], "Test-password-123"
|
||||
user = self.call("POST", "/api/v1/users", {
|
||||
"username": username, "display_name": "Unicode", "password": password,
|
||||
}, expected=201)
|
||||
session = self.call("POST", "/api/v1/auth/sessions", headers={
|
||||
"Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(),
|
||||
}, expected=201)
|
||||
auth = {"Authorization": "Bearer " + session["token"]}
|
||||
chat = self.call("POST", "/api/v1/chats", {
|
||||
"title": "Unicode", "member_ids": [user["id"]],
|
||||
}, auth, 201)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000]
|
||||
messages, keys = [], []
|
||||
for text in texts:
|
||||
key = uuid4().hex
|
||||
message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
|
||||
self.assertEqual(message["text"], text)
|
||||
messages.append(message)
|
||||
keys.append(key)
|
||||
for text in ("", "\0" * 4001, "🙂" * 4001):
|
||||
self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400)
|
||||
self.stop_server()
|
||||
self.start_server()
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
page = self.call("GET", path + "?limit=1" + (
|
||||
"&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else ""
|
||||
), headers=auth)
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
self.assertEqual(found, messages)
|
||||
if LAB >= 3:
|
||||
for text, message, key in zip(texts, messages, keys):
|
||||
replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
|
||||
self.assertEqual(replay, message)
|
||||
self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409)
|
||||
self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages))
|
||||
|
||||
def test_upgrade_preserves_old_text_and_is_repeatable(self):
|
||||
schema = "unicode_" + uuid4().hex
|
||||
with psycopg.connect(URL, autocommit=True) as conn:
|
||||
conn.execute(f'CREATE SCHEMA "{schema}"')
|
||||
isolated = make_conninfo(URL, options=f"-c search_path={schema}")
|
||||
try:
|
||||
old_text = "Старое 🙂 \\ literal \\u0000"
|
||||
with psycopg.connect(isolated) as conn:
|
||||
conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)")
|
||||
for migration in sorted((ROOT / "migrations").glob("*.sql")):
|
||||
version = int(migration.name.split("_", 1)[0])
|
||||
if version >= 9:
|
||||
continue
|
||||
conn.execute(migration.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,))
|
||||
user, chat, message = uuid4(), uuid4(), uuid4()
|
||||
conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,))
|
||||
conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,))
|
||||
conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user))
|
||||
conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text))
|
||||
for _ in range(2):
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
|
||||
with psycopg.connect(isolated) as conn:
|
||||
conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text())
|
||||
saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone()
|
||||
self.assertEqual(saved.decode("utf-8"), old_text)
|
||||
finally:
|
||||
with psycopg.connect(URL, autocommit=True) as conn:
|
||||
conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Prepare a private fingerprint, then verify a restored DB and API without source DB."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from src.auth import token_hash
|
||||
from src.db import database_url
|
||||
|
||||
|
||||
TABLES = {
|
||||
"users": "id",
|
||||
"chats": "id",
|
||||
"memberships": "chat_id,user_id",
|
||||
"messages": "id",
|
||||
"sessions": "token_hash",
|
||||
}
|
||||
|
||||
|
||||
def fingerprints(conn):
|
||||
result = {}
|
||||
for table, order in TABLES.items():
|
||||
digest = hashlib.sha256()
|
||||
count = 0
|
||||
with conn.cursor(name="fingerprint_" + table) as cursor:
|
||||
cursor.execute(f"SELECT * FROM {table} ORDER BY {order}")
|
||||
for row in cursor:
|
||||
digest.update(repr(tuple(row.values())).encode("utf-8"))
|
||||
digest.update(b"\n")
|
||||
count += 1
|
||||
result[table] = {"count": count, "sha256": digest.hexdigest()}
|
||||
return result
|
||||
|
||||
|
||||
def samples(conn, chat_id):
|
||||
found = []
|
||||
for offset in (0, 39_999, 79_999):
|
||||
row = conn.execute(
|
||||
"SELECT id,text FROM messages WHERE chat_id=%s "
|
||||
"ORDER BY created_at,id OFFSET %s LIMIT 1", (UUID(chat_id), offset)
|
||||
).fetchone()
|
||||
assert row is not None, "benchmark chat must contain 80000 messages"
|
||||
found.append({"id": str(row["id"]), "text_sha256": hashlib.sha256(row["text"]).hexdigest()})
|
||||
return found
|
||||
|
||||
|
||||
def request(url, token=None):
|
||||
headers = {"Authorization": "Bearer " + token} if token else {}
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=10) as response:
|
||||
return response.status, json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
|
||||
def prepare():
|
||||
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
|
||||
chat = conn.execute(
|
||||
"SELECT c.id,m.user_id FROM chats c JOIN memberships m ON m.chat_id=c.id "
|
||||
"WHERE c.title='Track 2 hot' ORDER BY c.created_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
assert chat is not None
|
||||
active = secrets.token_urlsafe(32)
|
||||
expired = secrets.token_urlsafe(32)
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(token_hash,user_id,expires_at) "
|
||||
"VALUES (%s,%s,clock_timestamp()+interval '1 hour'),"
|
||||
"(%s,%s,clock_timestamp()-interval '1 second')",
|
||||
(token_hash(active), chat["user_id"], token_hash(expired), chat["user_id"]),
|
||||
)
|
||||
conn.commit()
|
||||
expected = {
|
||||
"chat_id": str(chat["id"]), "active_token": active, "expired_token": expired,
|
||||
"tables": fingerprints(conn), "samples": samples(conn, str(chat["id"])),
|
||||
}
|
||||
print(json.dumps(expected)) # Redirect only into a temporary chmod-600 file.
|
||||
|
||||
|
||||
def verify():
|
||||
expected = json.load(sys.stdin)
|
||||
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
|
||||
actual_tables = fingerprints(conn)
|
||||
actual_samples = samples(conn, expected["chat_id"])
|
||||
assert actual_tables == expected["tables"], "restored table count/content mismatch"
|
||||
assert actual_samples == expected["samples"], "restored selected message mismatch"
|
||||
base = "http://127.0.0.1:8080"
|
||||
assert request(base + "/health/ready")[0] == 200
|
||||
status, page = request(base + "/api/v1/chats/" + expected["chat_id"] + "/messages?limit=50",
|
||||
expected["active_token"])
|
||||
assert status == 200
|
||||
assert expected["samples"][0]["id"] in {item["id"] for item in page["items"]}
|
||||
assert request(base + "/api/v1/chats/" + expected["chat_id"] + "/messages?limit=50",
|
||||
expected["expired_token"])[0] == 401
|
||||
print("Restore fingerprint PASS (all rows, ordered by primary key):")
|
||||
for table, value in actual_tables.items():
|
||||
print(f" {table}: count={value['count']} sha256={value['sha256']}")
|
||||
print("Selected synthetic messages PASS (id, text SHA-256):")
|
||||
for sample in actual_samples:
|
||||
print(f" {sample['id']} {sample['text_sha256']}")
|
||||
print("Restored API PASS: readiness 200, active token reads first message 200, expired token 401")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.argv[1:] == ["prepare"]:
|
||||
prepare()
|
||||
elif sys.argv[1:] == ["verify"]:
|
||||
verify()
|
||||
else:
|
||||
raise SystemExit("usage: test_restore.py prepare|verify")
|
||||
Reference in New Issue
Block a user