forked from mtusi2026/mtusi2026lab2
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01ecc8567e | ||
|
|
400e62ffdc |
+9
-18
@@ -1,20 +1,11 @@
|
||||
.git
|
||||
.github
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets
|
||||
certs
|
||||
volumes
|
||||
data
|
||||
backups
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.jks
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
.env*
|
||||
.venv*
|
||||
**/__pycache__
|
||||
tests
|
||||
student-tests
|
||||
evidence
|
||||
.DS_Store
|
||||
scripts
|
||||
secrets
|
||||
backups
|
||||
*.log
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: |
|
||||
python scripts/bootstrap.py
|
||||
docker compose up -d --build --wait
|
||||
make check test test-student
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-dependencies
|
||||
make research
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
@@ -15,6 +15,7 @@ data/
|
||||
backups/
|
||||
# Runtime/build artifacts
|
||||
.venv/
|
||||
.venv*/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
node_modules/
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2
|
||||
WORKDIR /app
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY src/ ./src/
|
||||
COPY migrations/ ./migrations/
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "src/server.py"]
|
||||
@@ -0,0 +1,24 @@
|
||||
ЛР 2 — БВТ2301 — Майорова — на 5
|
||||
|
||||
Автор: Майорова Софья, БВТ2301. Целевая оценка 5, трек 2 — план запроса и восстановление.
|
||||
|
||||
ЛР 1 отправлена: https://gitflic.ru/project/vmd/mtusi2026lab1/merge-request/140
|
||||
|
||||
Собственная реализация lab1 перенесена на PostgreSQL. Добавлены версионируемая миграция, PK/FK/UNIQUE, атомарное создание чата, ограниченный connection pool и таймауты. Регистрация использует scrypt, Basic выдаёт opaque Bearer-сессию; сессии с expiry и logout хранятся в PostgreSQL. Redis реально кэширует публичные профили с TTL; отказ кэша не отменяет проверку сессии в БД и не мешает работе.
|
||||
|
||||
Проверки: `make check validate`, `make test`, девять собственных тестов, сохранность данных и прежнего токена после рестарта API/PostgreSQL/Redis, expiry=2 с, logout, неправильный пароль, гонка username, rollback, отказы Redis/PostgreSQL и восстановление. Найденный собственными тестами OOM при параллельном scrypt исправлен ограничением KDF, повторный тест проходит при прежнем лимите 256 MiB.
|
||||
|
||||
Исследование на 100000 сообщений в десяти чатах разного размера: два индекса, EXPLAIN ANALYZE BUFFERS, пять повторов чтения, три измерения записи с COMMIT и WAL. Backup восстановлен в другой PostgreSQL-контейнер на новом volume; количества, пять выбранных сообщений и SHA-256 всей истории совпали. Полные планы, результаты и ограничения — в `evidence/index-study.json` и `REPORT.md`.
|
||||
|
||||
Исправлены замечания ревью: U+0000 сохраняется без потерь в display_name/title/text через UTF-8 BYTEA и миграцию 002; девятый тест проверяет полный round trip, кэш и границы длины. Проверен полный restart с NUL-содержащими строками. Restore ожидает TCP-готовность финального PostgreSQL; три новых независимых volume восстановлены с полной сверкой 100000 сообщений и контрольной строкой с U+0000. Вывод в evidence/review-*.
|
||||
|
||||
Запуск из чистого clone: `python3 scripts/bootstrap.py`, `docker compose up -d --build --wait`, `make check test test-student`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-dependencies`, `make research`.
|
||||
|
||||
- [x] Минимум предыдущей работы сохранён с переходом на сессии.
|
||||
- [x] Публичные контракты и tests не изменены.
|
||||
- [x] Фактический вывод проверок приложен.
|
||||
- [x] Полный restart, ошибки зависимостей и трек 5 проверены.
|
||||
- [x] Пароли, tokens, volumes и backup находятся вне Git.
|
||||
- [x] README задания сохранён, REPORT.md заполнен.
|
||||
|
||||
Ограничения: один PostgreSQL; нет HA/PITR; Redis — disposable cache. Локальный HTTP предназначен для синтетического стенда. Codex использован при реализации, экспериментах и составлении отчёта.
|
||||
@@ -21,6 +21,16 @@ test:
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: test-student test-dependencies research
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_api.py -v
|
||||
|
||||
test-dependencies:
|
||||
$(PYTHON) student-tests/dependencies.py
|
||||
|
||||
research:
|
||||
$(PYTHON) student-tests/research.py
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -1,37 +1,169 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: **Майорова Софья, БВТ2301**. Целевая оценка: **5**, трек 2 «План запроса и восстановление».
|
||||
|
||||
Ветка: `solution/BVT2301-Mayorova`. Заголовок МР: `ЛР 2 — БВТ2301 — Майорова — на 5`.
|
||||
Текущий commit: `git log -1 --oneline`.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine с Compose v2+ и Linux-контейнерами, Python 3.9+, make. Исходники, собственные тесты и HTTP-слой перенесены из своей lab1; контракты и публичные тесты взяты из skeleton **lab2**, без замены на старые версии.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
python3 scripts/bootstrap.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait
|
||||
make check test test-student
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-dependencies
|
||||
make research
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Bootstrap один раз создаёт `.env` и случайные локальные secrets; повтор не перезаписывает пароль или БД. Никаких вручную установленных PostgreSQL/Redis не требуется. API сам применяет версии миграций при запуске, до открытия HTTP-порта. Для нового порта задайте `APP_PORT=18086 docker compose up -d --wait` и передайте соответствующий `BASE_URL` тестам. Сценарий expiry временно занимает 18083, исследование не публикует порт PostgreSQL.
|
||||
|
||||
Полная валидация OpenAPI:
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
Остановка с сохранением данных: `docker compose down`; повторный запуск — `docker compose up -d --wait`.
|
||||
**Намеренное удаление всех учебных данных основного стенда**: `docker compose down -v`. Это не команда restart/persistence-проверки. Файлы backup сохраняются отдельно в игнорируемом каталоге `backups/`.
|
||||
Исследование дополнительно оставляет отдельный остановленный PostgreSQL и собственный том восстановления; их имена записаны в `evidence/index-study.json`. Для их намеренного удаления используйте `docker rm <restore_container>` и `docker volume rm <fresh_restore_volume>` только после проверки этих имён. Не выполняйте общий `docker system prune`.
|
||||
|
||||
## Архитектура, схема и границы транзакций
|
||||
|
||||
`Клиент → API → PostgreSQL`; Redis кэширует неизменяемые публичные профили `User`. PostgreSQL — источник истины для пользователей, чатов, участников, сообщений и сессий. История сообщений никогда не берётся из Redis: успешный COMMIT виден следующему чтению. Удаления/редактирования профилей в контракте нет, поэтому cache-aside профиля не нарушает read-after-write истории.
|
||||
|
||||
Схема в `migrations/001_initial.sql`; миграция `002_unicode_nul.sql` сохраняет существующие строки и переводит display_name/title/text в UTF-8 BYTEA:
|
||||
|
||||
- `users`: UUID PK, UNIQUE username, display_name, scrypt hash, server timestamp.
|
||||
- `chats`: UUID PK, title, created_at, last_message_at.
|
||||
- `chat_members`: составной PK (chat_id, user_id), FK в chats/users, обратный индекс (user_id, chat_id).
|
||||
- `messages`: UUID PK, FK в chats/users, составной FK (chat_id, sender_id) в членство; индекс (chat_id, created_at, id).
|
||||
- `sessions`: PK SHA-256 случайного токена, FK user_id, expiry и индекс expiry. SHA-256 здесь индексирует высокоэнтропийный токен, **не пароль**.
|
||||
- `schema_migrations`: применённые имена версий. Advisory transaction lock исключает одновременное применение миграции несколькими API.
|
||||
|
||||
PostgreSQL TEXT не поддерживает U+0000, хотя контракт допускает его в Unicode-строке. Три пользовательских поля хранятся как UTF-8 bytes в BYTEA, на границе API кодируются/декодируются без потерь. Python проверяет длину в code points (100/100/4000); ограничения БД дополнительно ограничивают UTF-8 размер 400/400/16000 байт. Username, password_hash и данные сессий остаются TEXT. Миграция конвертирует прежние строки через convert_to. [TEXT](https://www.postgresql.org/docs/17/datatype-character.html), [BYTEA](https://www.postgresql.org/docs/17/datatype-binary.html).
|
||||
|
||||
Создание чата, проверка всех участников и вставка членства — одна транзакция. Ошибка неизвестного участника после INSERT чата откатывает весь составной объект; собственный тест проверяет отсутствие даже невидимого orphan через SQL COUNT.
|
||||
Запись сообщения и увеличение `last_message_at` — одна транзакция. UPDATE строки чата сериализует timestamp внутри чата; сервер выбирает max(часы, предыдущий timestamp + 1 мкс). Это сохраняет порядок keyset при конкурентных отправках и переводе часов назад. Уровень изоляции — PostgreSQL READ COMMITTED; snapshot между HTTP-страницами не обещается.
|
||||
|
||||
SQL-параметры передаются отдельно от текста запросов. Connection pool: min=1, max=8, ожидание 2 с; connect timeout 2 с, statement timeout 2 с, lock timeout 1,5 с. Ошибки зависимости ограничены JSON 503, внутренний SQL не возвращается клиенту. Чужие объекты скрыты 404. Readiness зависит от PostgreSQL; liveness — только от процесса.
|
||||
|
||||
## Аутентификация и Redis
|
||||
|
||||
Регистрация принимает 12–128 Unicode code points пароля, включая двоеточие и emoji. KDF — библиотечный `hashlib.scrypt`: N=16384, r=8, p=1, 16-байтовая случайная соль, dklen=32. Хранятся алгоритм/параметры/соль/hash. Сравнение через `hmac.compare_digest`; для неизвестного пользователя при допустимом формате credentials вычисляется dummy hash. Неверные credentials возвращают одинаковый 401 с Basic challenge. Профили не содержат паролей/hash.
|
||||
|
||||
Basic разрешён только на `POST /auth/sessions`, без JSON body. Токен — `secrets.token_urlsafe(32)`, 256 бит энтропии. На бизнес-endpoint принимается Bearer; `X-User-Id` не меняет владельца и не даёт доступ. На **каждом** защищённом запросе PostgreSQL проверяет действительность сессии и expiry. Logout удаляет сессию, поэтому даже тёплый кэш профиля не даёт повторного доступа. Ещё действующие сессии переживают рестарт, TTL не продлевается.
|
||||
|
||||
Redis: cache-aside `user:<UUID>`, TTL 60 с, лимит 32 MiB и allkeys-lru. При miss профиль читается из PostgreSQL; Redis не хранит bearer token и не определяет аутентификацию. При недоступности Redis с таймаутом 150 мс запрос безопасно идёт в PostgreSQL. После рестарта кэш прогревается повторно. Его persistence намеренно выключен: обязательных данных в Redis нет. Если в будущем появится редактирование User, потребуется инвалидация `user:<UUID>` после COMMIT.
|
||||
|
||||
Собственный конкурентный тест выявил OOM при 10 параллельных scrypt в контейнере 256 MiB (EXIT 137). Исправлено ограничением KDF до двух одновременных вычислений и `MALLOC_ARENA_MAX=2`; ожидание KDF ограничено 1,5 с, далее 503. Тот же тест под **тем же** лимитом памяти теперь проходит: один 201, девять 409. Доказательства — `evidence/kdf-regression.json`, `evidence/student-tests.txt`.
|
||||
|
||||
## Компоненты и конфигурация
|
||||
|
||||
| Роль / service | Версия | Health / readiness | Хранение / сеть |
|
||||
| --- | --- | --- | --- |
|
||||
| API / `api` | Python 3.12.8, digest в Dockerfile | `/health/live`, `/health/ready`, Docker healthcheck | UID 10001; без локального состояния; только 127.0.0.1:8080 опубликован |
|
||||
| PostgreSQL / `postgres` | 17.2-bookworm, digest в compose.yaml | pg_isready и SQL SELECT 1 через API | именованный `postgres_data`, порт только внутри Compose |
|
||||
| Redis / `redis` | 7.4.1-bookworm, digest в compose.yaml | redis-cli ping; не критичен для readiness API | только внутренний порт; кэш disposable |
|
||||
|
||||
API ограничен 1 CPU / 256 MiB; PostgreSQL — 1 CPU / 512 MiB; Redis — 0,5 CPU / 64 MiB. Окружение измерения: macOS arm64, Linux Colima VM 4 vCPU / 4 GiB. Python-зависимости и транзитивный typing-extensions закреплены в `requirements.txt`.
|
||||
|
||||
`APP_HOST=0.0.0.0` и внутренний APP_PORT=8080 заданы Dockerfile. `APP_PORT` из `.env` задаёт опубликованный порт Compose. `APP_ENV` — информационная метка skeleton. `SESSION_TTL_SECONDS` реально передаётся API, default 3600; нормальные smoke/persistence выполняются с TTL ≥300. `DATABASE_URL_FILE`/`CACHE_URL_FILE` указывают на Docker secrets. `SESSION_STORE=postgres` из исходного примера соответствует выбранному решению; переключение backend по этой переменной не реализовано.
|
||||
|
||||
`secrets/` имеет host mode 0700; пароль PostgreSQL — 0600. Файлы URL имеют 0644 внутри защищённого host-каталога, чтобы Compose мог смонтировать их для UID 10001. Secrets, `.env`, volumes и backups исключены из Git и build context. В публичном отчёте нет токенов и паролей. Локальный учебный HTTP и доверенная внутренняя сеть не заменяют TLS и ограничение DB-role для реального сервиса.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
После исправлений выполнен новый чистый локальный git clone, сборка и запуск на новом пустом volume. Применены обе миграции 001/002; публичный smoke и все девять собственных тестов проходят: `evidence/review-clean-clone.txt`. Первичный прогон до ревью сохранён в `evidence/clean-clone.txt`. Сохранность и отказы текущей версии дополнительно проверены: `evidence/review-persistence.txt`, `evidence/review-dependencies.txt`.
|
||||
|
||||
| Критерий | Команда / сценарий | Реальный результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Offline / OpenAPI | `make check`, `make validate` | PASS | `evidence/check.txt`, `evidence/validation.txt` |
|
||||
| Обязательный API | `make test` | PASS lab2 | `evidence/smoke.txt` |
|
||||
| Негативные сценарии и Unicode | `make test-student` | 9 тестов, все PASS, включая U+0000 | `evidence/review-tests.txt` |
|
||||
| Конкурентные сообщения / cursor | тот же набор, 100 POST / 10 клиентов | 100 уникальных ID, без потерь | тот же файл |
|
||||
| Гонка username | тот же набор | ровно один 201, девять 409 | тот же файл |
|
||||
| FK/UNIQUE/миграции с нуля | первый `compose up`, повторные старты | версии 001/002 применены; исследование создаёт схему с нуля | `migrations/`, `evidence/review-restore-runs.json` |
|
||||
| Атомарность составного чата | `make test-dependencies`, SQL COUNT до/после | rollback, количество chats не изменилось | `evidence/dependencies.json` |
|
||||
| Сохранность данных и прежнего токена | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS после рестарта API + PostgreSQL + Redis | `evidence/persistence.txt` |
|
||||
| Реальность restart hook | сравнение StartedAt и volumes всех трёх services | процессы новые по времени старта, том PostgreSQL тот же | `evidence/restart-before.json`, `evidence/restart-after.json` |
|
||||
| Реальный Redis | DEL профиля, затем GET /users/me и Redis TTL | ключ реально заполнен, TTL 60 с | `evidence/dependencies.json` |
|
||||
| Expiry | отдельный API, TTL=2, ожидание 2,2 с | прежний Bearer → 401 | тот же файл |
|
||||
| Logout / неверный пароль / Basic только на login | smoke + собственный тест | ожидаемые 401/204 | smoke и student-tests |
|
||||
| Отказ Redis | остановка redis, чтение/запись с прежним токеном | работа продолжается, поддельный Bearer → 401 | dependencies |
|
||||
| Отказ PostgreSQL | остановка postgres | me/ready → 503; live → 200; после возврата прежний токен работает | dependencies |
|
||||
| Индекс ≥10000 сообщений | EXPLAIN ANALYZE BUFFERS на 100000 | keyset использует составной индекс | `evidence/index-study.json` |
|
||||
| Трек 5: ≥2 индекса, запись | `make research` | 100000 сообщений, 2 индекса, 5 повторов чтения и 3 записи | тот же файл |
|
||||
| Трек 5: backup → новое хранилище | pg_dump/pg_restore в отдельный контейнер и новый volume | count, 5 образцов и SHA-256 всей истории совпадают | тот же файл, `evidence/research.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Исследование индекса и восстановления
|
||||
|
||||
## Эксперименты и отказы
|
||||
Гипотеза: индекс только по chat_id заставляет отфильтровывать/сортировать значительную часть большого чата при глубоком cursor. B-tree `(chat_id, created_at, id)` позволит читать limit+1 сразу из нужного диапазона; за это придётся платить размером индекса и записью WAL.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
`student-tests/index_study.py` создаёт **отдельную синтетическую БД**, не заполняет рабочие чаты тестового API. Размеры десяти чатов: **70000, 20000, 5000, 2000, 1000, 800, 500, 300, 200, 200** — суммарно 100000. Текст — 64 байта UTF-8. Одна контрольная строка содержит U+0000 и имеет UUID=0, чтобы обязательно попасть в образцы backup/restore; остальные UUID случайные. Timestamp с шагом 1 мкс. Сравниваются `messages_history(chat_id)` и `messages_history(chat_id, created_at, id)`; PK(id) и прочие ограничения одинаковы.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Запрос совпадает с API keyset: фильтр chat_id и `(created_at,id)>cursor`, ORDER BY created_at,id, LIMIT 51. Для чатов 70000, 20000 и 200 cursor ставится на 80% истории. Перед замером — ANALYZE и один прогрев; затем пять `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)`, без отключения seqscan и других подсказок планировщику. Сохранены все планы, включая buffers. Поочерёдно меняется один индекс.
|
||||
|
||||
| Индекс | Сообщений в чате | Медиана Execution Time, мс |
|
||||
| --- | --- | --- |
|
||||
| chat_only | 70000 | 4.463 |
|
||||
| chat_only | 20000 | 1.273 |
|
||||
| chat_only | 200 | 0.020 |
|
||||
| composite | 70000 | 0.011 |
|
||||
| composite | 20000 | 0.011 |
|
||||
| composite | 200 | 0.008 |
|
||||
|
||||
Для большого чата простой индекс приводит к полному/широкому проходу и сортировке, составной — к ограниченному индексному диапазону. Для маленького чата разница близка к шуму. Сам API использует составной индекс из миграции.
|
||||
|
||||
Стоимость записи: для каждого варианта три отдельных COPY по 1000 новых сообщений в большой чат, время от COPY до завершения COMMIT; UUID подготовлены до таймера. После каждого повтора добавленные сообщения удаляются и фиксируются отдельно; cleanup не входит в измерение. WAL — разница `pg_current_wal_insert_lsn()` до/после. Все исходные измерения сохранены.
|
||||
|
||||
| Индекс | Размер индекса, байт | Медиана COPY+COMMIT 1000 строк, мс | Медиана WAL, байт |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| chat_only | 753664 | 9.849 | 241648 |
|
||||
| composite | 6127616 | 9.935 | 311584 |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Большой индекс ускоряет чтение, но увеличивает занимаемое место и WAL. Время записи в этих коротких последовательных прогонах заметно колеблется, на него влияют checkpoint, кэш, COPY, CPU quota и фоновые процессы; три повтора не дают универсального коэффициента замедления. Порядок вариантов фиксирован, cold-cache I/O не исследовался. Это анализ query plan и компромисса индекса, не измерение HTTP RPS всего приложения.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
### Backup и новое хранилище
|
||||
|
||||
`student-tests/research.py`:
|
||||
|
||||
1. Запускает исследование в отдельной БД; делает `pg_dump -Fc` согласованного снимка.
|
||||
2. Создаёт **другой PostgreSQL-контейнер с новым именованным volume**, тем же закреплённым образом и собственным каталогом данных; порты хоста не публикуются.
|
||||
3. Ожидает TCP-готовность на 127.0.0.1, затем выполняет `pg_restore -h 127.0.0.1 --exit-on-error` в новую пустую БД. Временный Unix-socket сервер инициализации не проходит эту проверку.
|
||||
4. Сравнивает количества users/chats/chat_members/messages/sessions, пять выбранных сообщений и SHA-256 **всех 100000 сообщений** в порядке UUID, включая поля сообщений. В research fixture sessions=0; сохранность действующей сессии отдельно доказана persistence-тестом.
|
||||
5. Сохраняет JSON-сводку, останавливает restore-контейнер; backup и новый volume оставляет для проверки.
|
||||
|
||||
Размер dump: 2655198 байт. Dump: 0.280 с; bootstrap нового PostgreSQL: 0.820 с; pg_restore: 0.267 с. Bootstrap + восстановление: 1.087 с, без последующей проверки checksum. Это локальный учебный набор, не RTO большого сервиса.
|
||||
|
||||
Исходный volume: `mtusi-lab2_postgres_data`; новый: `mtusi-lab2-restore_4bc6680354a1-data`. Разные ID контейнеров и полный результат сверки — в JSON. Backup не коммитится.
|
||||
Восстановленный SHA-256 истории: `8c3e317ab75daa8a5e97641eac51e3a349cffe2293bb83bc2ba22ce602cbf112`.
|
||||
|
||||
### Отказы и ограничения durability
|
||||
|
||||
При потере Redis обязательных данных не теряется, кэш восстановим. В тесте чтение/запись/проверки при выключенном Redis заняли 0.050 с суммарно. При выключенном PostgreSQL первый защищённый запрос вернул 503 за 0.004 с; анонимный доступ не разрешался. После возврата зависимостей исходная сессия работала.
|
||||
|
||||
Рестарт контейнера с сохранённым volume не теряет подтверждённые данные при обычных настройках PostgreSQL fsync/synchronous_commit. Он не доказывает устойчивость к поломке диска/host или потере volume. Резервная копия даёт состояние на момент snapshot pg_dump: сообщения после этого снимка будут потеряны при восстановлении только из данного backup. PITR/WAL archiving, репликация PostgreSQL и HA не реализованы. Readiness честно становится 503 без БД.
|
||||
|
||||
Учебные ограничения: один PostgreSQL и один API; DB-role bootstrap владеет учебной БД, ограничение его production-прав не сделано. Накопленные истёкшие сессии не обслуживаются, но фоновый cleanup таблицы пока не добавлен. Redis не кэширует историю и не сокращает обязательную проверку сессии в БД. Общий HTTP-слой не предназначен для публичного Internet-deployment без TLS/reverse proxy.
|
||||
|
||||
## Исправления по замечаниям преподавателя
|
||||
|
||||
- U+0000: миграция 002 и lossless UTF-8 BYTEA для трёх строковых полей. Новый black-box тест проверяет NUL, буквальную последовательность backslash-u0000, emoji, границу 4000 code points, профиль из login/БД/Redis, title и чтение истории. Девять собственных тестов и публичный smoke проходят: `evidence/review-tests.txt`.
|
||||
- Сохранность: профиль, чат и сообщение с U+0000 без изменений прочитаны после рестарта API/PostgreSQL/Redis, прежний Bearer действителен: `evidence/review-unicode-persistence.txt`.
|
||||
- Гонка init-сервера: readiness и restore используют TCP. Три последовательных полных исследования с новыми контейнерами и томами прошли; каждый раз совпали counts, выбранные строки (включая NUL) и SHA-256 всей истории: `evidence/review-research.txt`, `evidence/review-restore-runs.json`. Метрики выше обновлены по последнему прогону текущей схемы.
|
||||
|
||||
## Коротко для защиты
|
||||
|
||||
COMMIT подтверждает транзакцию; устойчивость зависит от WAL, fsync, synchronous_commit и сохранности носителя. Аутентификация определяет пользователя по Bearer, членство отдельно определяет доступ к чату. Flush/restart Redis затрагивает только кэш профилей; сессии остаются в PostgreSQL. Password KDF намеренно затратен и использует соль, быстрый checksum для паролей не подходит.
|
||||
|
||||
## Источники и помощь
|
||||
|
||||
Исходный skeleton lab2, его контракты и публичные tests; собственные src/common.py, src/server.py и тесты из lab1.
|
||||
Документация: [PostgreSQL EXPLAIN](https://www.postgresql.org/docs/17/using-explain.html), [pg_dump](https://www.postgresql.org/docs/17/app-pgdump.html), [pg_restore](https://www.postgresql.org/docs/17/app-pgrestore.html), [psycopg pool](https://www.psycopg.org/psycopg3/docs/advanced/pool.html), [Python scrypt](https://docs.python.org/3.12/library/hashlib.html#hashlib.scrypt), [Redis EXPIRE](https://redis.io/docs/latest/commands/expire/), [RFC 7617](https://www.rfc-editor.org/rfc/rfc7617).
|
||||
Codex использован для реализации, тестирования, экспериментов и отчёта. Все численные результаты получены локальными запусками; контракты и публичные проверки не ослаблялись.
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: mtusi-lab2
|
||||
services:
|
||||
api:
|
||||
build: .
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
mem_limit: 256m
|
||||
cpus: 1
|
||||
stop_grace_period: 10s
|
||||
postgres:
|
||||
image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
mem_limit: 512m
|
||||
cpus: 1
|
||||
redis:
|
||||
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
|
||||
command: ["redis-server", "--maxmemory", "32mb", "--maxmemory-policy", "allkeys-lru", "--save", "", "--appendonly", "no"]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
mem_limit: 64m
|
||||
cpus: 0.5
|
||||
volumes:
|
||||
postgres_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
@@ -0,0 +1,78 @@
|
||||
Image mtusi-lab2-api Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 656B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 739B 0.1s done
|
||||
#2 DONE 0.1s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#3 DONE 0.1s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 132B 0.0s done
|
||||
#4 DONE 0.1s
|
||||
|
||||
#5 [internal] load build context
|
||||
#5 transferring context: 24.73kB 0.2s done
|
||||
#5 DONE 0.3s
|
||||
|
||||
#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 0.4s done
|
||||
#6 DONE 0.5s
|
||||
|
||||
#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
#7 CACHED
|
||||
|
||||
#8 [4/7] COPY requirements.txt ./
|
||||
#8 CACHED
|
||||
|
||||
#9 [2/7] WORKDIR /app
|
||||
#9 CACHED
|
||||
|
||||
#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt
|
||||
#10 CACHED
|
||||
|
||||
#11 [6/7] COPY src/ ./src/
|
||||
#11 DONE 0.2s
|
||||
|
||||
#12 [7/7] COPY migrations/ ./migrations/
|
||||
#12 DONE 0.3s
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers
|
||||
#13 exporting layers 1.1s done
|
||||
#13 exporting manifest sha256:5017231aa6022853d9ca872aa39e5c980e31c40e17c7986757b3afc3651064b5 0.1s done
|
||||
#13 exporting config sha256:2b594068234b84afae7062ee6b719d9e84a772923d9a4f0d9c2ab431aac7f09c
|
||||
#13 exporting config sha256:2b594068234b84afae7062ee6b719d9e84a772923d9a4f0d9c2ab431aac7f09c 0.1s done
|
||||
#13 exporting attestation manifest sha256:3ae47f10c02268299f427294491ad4f25e5485ce4905715abe5868d2d0818ca6
|
||||
#13 exporting attestation manifest sha256:3ae47f10c02268299f427294491ad4f25e5485ce4905715abe5868d2d0818ca6 0.2s done
|
||||
#13 exporting manifest list sha256:90f48f95ec551e4de0cb9ca3c8da3fbac9bfa588a52cd6fd9b21a225ac9e48cd
|
||||
#13 exporting manifest list sha256:90f48f95ec551e4de0cb9ca3c8da3fbac9bfa588a52cd6fd9b21a225ac9e48cd 0.1s done
|
||||
#13 naming to docker.io/library/mtusi-lab2-api:latest
|
||||
#13 naming to docker.io/library/mtusi-lab2-api:latest 0.0s done
|
||||
#13 unpacking to docker.io/library/mtusi-lab2-api:latest
|
||||
#13 unpacking to docker.io/library/mtusi-lab2-api:latest 0.2s done
|
||||
#13 DONE 2.1s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.1s
|
||||
Image mtusi-lab2-api Built
|
||||
Container mtusi-lab2-redis-1 Running
|
||||
Container mtusi-lab2-postgres-1 Running
|
||||
Container mtusi-lab2-api-1 Recreate
|
||||
Container mtusi-lab2-api-1 Recreated
|
||||
Container mtusi-lab2-postgres-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Healthy
|
||||
Container mtusi-lab2-postgres-1 Healthy
|
||||
Container mtusi-lab2-api-1 Starting
|
||||
Container mtusi-lab2-api-1 Started
|
||||
Container mtusi-lab2-redis-1 Waiting
|
||||
Container mtusi-lab2-api-1 Waiting
|
||||
Container mtusi-lab2-postgres-1 Waiting
|
||||
Container mtusi-lab2-postgres-1 Healthy
|
||||
Container mtusi-lab2-redis-1 Healthy
|
||||
Container mtusi-lab2-api-1 Healthy
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,130 @@
|
||||
$ git clone --local /Users/ssmayorova/Documents/ChatGPT/Высоконагруженные приложения/mtusi2026lab2 /Users/ssmayorova/Documents/ChatGPT/Высоконагруженные приложения/clean-clone-o_gf7ofj/mtusi2026lab2
|
||||
Cloning into '/Users/ssmayorova/Documents/ChatGPT/Высоконагруженные приложения/clean-clone-o_gf7ofj/mtusi2026lab2'...
|
||||
done.
|
||||
$ python3 scripts/bootstrap.py
|
||||
Local configuration ready; secrets were not printed.
|
||||
$ docker compose up -d --build --wait
|
||||
Image mtusi-clean-144e2e11-api Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 718B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 739B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 132B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [internal] load build context
|
||||
#5 transferring context: 24.62kB 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 0.1s done
|
||||
#6 DONE 0.1s
|
||||
|
||||
#7 [4/7] COPY requirements.txt ./
|
||||
#7 CACHED
|
||||
|
||||
#8 [5/7] RUN pip install --no-cache-dir -r requirements.txt
|
||||
#8 CACHED
|
||||
|
||||
#9 [2/7] WORKDIR /app
|
||||
#9 CACHED
|
||||
|
||||
#10 [6/7] COPY src/ ./src/
|
||||
#10 CACHED
|
||||
|
||||
#11 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
#11 CACHED
|
||||
|
||||
#12 [7/7] COPY migrations/ ./migrations/
|
||||
#12 CACHED
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers done
|
||||
#13 exporting manifest sha256:17e1aa1e1ab70d739372aac14340c7082aa2b9d516783f5bb8416e85ac608009 0.0s done
|
||||
#13 exporting config sha256:08162a60f623a2176f3cf1137372f23710c41dd7f958d7212129d15d5829c02b
|
||||
#13 exporting config sha256:08162a60f623a2176f3cf1137372f23710c41dd7f958d7212129d15d5829c02b 0.0s done
|
||||
#13 exporting attestation manifest sha256:ed2d5c62a3502802fd1706fca353486999cc539bd9b0d7f4171309a19a8fabe3 0.0s done
|
||||
#13 exporting manifest list sha256:f866af8b5e84486fa48b2a6264eb645a81e4e41773a66380f849fb3628f59f49 0.0s done
|
||||
#13 naming to docker.io/library/mtusi-clean-144e2e11-api:latest done
|
||||
#13 unpacking to docker.io/library/mtusi-clean-144e2e11-api:latest 0.0s done
|
||||
#13 DONE 0.2s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
Image mtusi-clean-144e2e11-api Built
|
||||
Network mtusi-clean-144e2e11_default Creating
|
||||
Network mtusi-clean-144e2e11_default Created
|
||||
Volume mtusi-clean-144e2e11_postgres_data Creating
|
||||
Volume mtusi-clean-144e2e11_postgres_data Created
|
||||
Container mtusi-clean-144e2e11-redis-1 Creating
|
||||
Container mtusi-clean-144e2e11-postgres-1 Creating
|
||||
Container mtusi-clean-144e2e11-postgres-1 Created
|
||||
Container mtusi-clean-144e2e11-redis-1 Created
|
||||
Container mtusi-clean-144e2e11-api-1 Creating
|
||||
Container mtusi-clean-144e2e11-api-1 Created
|
||||
Container mtusi-clean-144e2e11-redis-1 Starting
|
||||
Container mtusi-clean-144e2e11-postgres-1 Starting
|
||||
Container mtusi-clean-144e2e11-redis-1 Started
|
||||
Container mtusi-clean-144e2e11-postgres-1 Started
|
||||
Container mtusi-clean-144e2e11-postgres-1 Waiting
|
||||
Container mtusi-clean-144e2e11-redis-1 Waiting
|
||||
Container mtusi-clean-144e2e11-redis-1 Healthy
|
||||
Container mtusi-clean-144e2e11-postgres-1 Healthy
|
||||
Container mtusi-clean-144e2e11-api-1 Starting
|
||||
Container mtusi-clean-144e2e11-api-1 Started
|
||||
Container mtusi-clean-144e2e11-redis-1 Waiting
|
||||
Container mtusi-clean-144e2e11-api-1 Waiting
|
||||
Container mtusi-clean-144e2e11-postgres-1 Waiting
|
||||
Container mtusi-clean-144e2e11-redis-1 Healthy
|
||||
Container mtusi-clean-144e2e11-postgres-1 Healthy
|
||||
Container mtusi-clean-144e2e11-api-1 Healthy
|
||||
$ make check test test-student BASE_URL=http://localhost:18086
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://localhost:18086"
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
BASE_URL="http://localhost:18086" python3 -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 8 tests in 11.843s
|
||||
|
||||
OK
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18086
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18086"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ docker compose exec -T postgres psql -U messenger -d messenger -Atc SELECT count(*) FROM schema_migrations;
|
||||
1
|
||||
Container mtusi-clean-144e2e11-api-1 Stopping
|
||||
Container mtusi-clean-144e2e11-api-1 Stopped
|
||||
Container mtusi-clean-144e2e11-api-1 Removing
|
||||
Container mtusi-clean-144e2e11-api-1 Removed
|
||||
Container mtusi-clean-144e2e11-postgres-1 Stopping
|
||||
Container mtusi-clean-144e2e11-redis-1 Stopping
|
||||
Container mtusi-clean-144e2e11-redis-1 Stopped
|
||||
Container mtusi-clean-144e2e11-redis-1 Removing
|
||||
Container mtusi-clean-144e2e11-postgres-1 Stopped
|
||||
Container mtusi-clean-144e2e11-postgres-1 Removing
|
||||
Container mtusi-clean-144e2e11-redis-1 Removed
|
||||
Container mtusi-clean-144e2e11-postgres-1 Removed
|
||||
Volume mtusi-clean-144e2e11_postgres_data Removing
|
||||
Network mtusi-clean-144e2e11_default Removing
|
||||
Volume mtusi-clean-144e2e11_postgres_data Removed
|
||||
Network mtusi-clean-144e2e11_default Removed
|
||||
PASS: clean local clone, fresh image and all requested acceptance tests.
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"composite_operation_rollback": true,
|
||||
"redis_user_cache_ttl_seconds": 60,
|
||||
"redis_outage_seconds": 0.05048745900000007,
|
||||
"redis_recovered_without_relogin": true,
|
||||
"postgres_outage_response_seconds": 0.004356375000000412,
|
||||
"postgres_recovered_without_relogin": true,
|
||||
"short_ttl_expired": true
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
Container mtusi-lab2-redis-1 Stopping
|
||||
Container mtusi-lab2-redis-1 Stopped
|
||||
Container mtusi-lab2-redis-1 Starting
|
||||
Container mtusi-lab2-redis-1 Started
|
||||
Container mtusi-lab2-redis-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Healthy
|
||||
Container mtusi-lab2-postgres-1 Stopping
|
||||
Container mtusi-lab2-postgres-1 Stopped
|
||||
Container mtusi-lab2-postgres-1 Starting
|
||||
Container mtusi-lab2-postgres-1 Started
|
||||
Container mtusi-lab2-postgres-1 Waiting
|
||||
Container mtusi-lab2-postgres-1 Healthy
|
||||
Container mtusi-lab2-expiry-f1a39764 Creating
|
||||
Container mtusi-lab2-expiry-f1a39764 Created
|
||||
PASS: rollback, real Redis TTL/cache, Redis outage, PostgreSQL outage/recovery and session expiry.
|
||||
@@ -0,0 +1,18 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
BASE_URL="http://localhost:8080" python3 -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 8 tests in 13.427s
|
||||
|
||||
OK
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"scenario": "Initial own test: 10 concurrent registrations, before bounding KDF concurrency",
|
||||
"container_memory_mib": 256,
|
||||
"observed_exit_code": 137,
|
||||
"observed_oom_killed": true,
|
||||
"fix": "Limit simultaneous scrypt computations to 2 and set MALLOC_ARENA_MAX=2; rerun the same registration race under unchanged memory limit",
|
||||
"final_result_source": "student-tests.txt"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,6 @@
|
||||
Container mtusi-lab2-api-run-3b62fd7eb732 Creating
|
||||
Container mtusi-lab2-api-run-3b62fd7eb732 Created
|
||||
Container mtusi-lab2-api-run-0145f8a8aff1 Creating
|
||||
Container mtusi-lab2-api-run-0145f8a8aff1 Created
|
||||
PASS: 100000 messages; two indexes; write costs; backup/restore counts, samples and full checksum match.
|
||||
Backup seconds: 1.102 Restore seconds: 1.742
|
||||
@@ -0,0 +1,27 @@
|
||||
[
|
||||
{
|
||||
"service": "api",
|
||||
"container_id": "6b9b8576b1cd",
|
||||
"started_at": "2026-10-23T09:20:44.739602998Z",
|
||||
"status": "running",
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container_id": "07430ea862b6",
|
||||
"started_at": "2026-10-23T09:20:41.28275813Z",
|
||||
"status": "running",
|
||||
"volumes": [
|
||||
"mtusi-lab2_postgres_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container_id": "5fecf0c5edd1",
|
||||
"started_at": "2026-10-23T09:20:41.313335565Z",
|
||||
"status": "running",
|
||||
"volumes": [
|
||||
"b72dab0a9ef82932abd5655e09a614b5c99b3a7cef959426f05ad0a2ed7dbc79"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,27 @@
|
||||
[
|
||||
{
|
||||
"service": "api",
|
||||
"container_id": "6b9b8576b1cd",
|
||||
"started_at": "2026-10-23T09:19:48.119673288Z",
|
||||
"status": "running",
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container_id": "07430ea862b6",
|
||||
"started_at": "2026-10-23T09:19:48.24800751Z",
|
||||
"status": "running",
|
||||
"volumes": [
|
||||
"mtusi-lab2_postgres_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container_id": "5fecf0c5edd1",
|
||||
"started_at": "2026-10-23T09:19:48.06854916Z",
|
||||
"status": "running",
|
||||
"volumes": [
|
||||
"b72dab0a9ef82932abd5655e09a614b5c99b3a7cef959426f05ad0a2ed7dbc79"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,75 @@
|
||||
Image mtusi-lab2-api Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 656B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 739B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 132B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [internal] load build context
|
||||
#5 transferring context: 12.71kB done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [4/7] COPY requirements.txt ./
|
||||
#7 CACHED
|
||||
|
||||
#8 [2/7] WORKDIR /app
|
||||
#8 CACHED
|
||||
|
||||
#9 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
#9 CACHED
|
||||
|
||||
#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt
|
||||
#10 CACHED
|
||||
|
||||
#11 [6/7] COPY src/ ./src/
|
||||
#11 DONE 0.0s
|
||||
|
||||
#12 [7/7] COPY migrations/ ./migrations/
|
||||
#12 DONE 0.0s
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers 0.0s done
|
||||
#13 exporting manifest sha256:4629666110b9f12ff95b707e9aafdd14712e1eacef6918851f271840c4a0a7dc done
|
||||
#13 exporting config sha256:7c4d81b7399c8d2e57ab11ad9087b262cbdc4f79ce3298db7087079e530ce18a done
|
||||
#13 exporting attestation manifest sha256:7e311d74504e1fe4150d50a931487c137683dce1c5ae5c6779c0493722acd01a done
|
||||
#13 exporting manifest list sha256:0712b708d3a29530642a2c22dea53064fbfdfedc9537a18f3a04ba343e11da7f done
|
||||
#13 naming to docker.io/library/mtusi-lab2-api:latest done
|
||||
#13 unpacking to docker.io/library/mtusi-lab2-api:latest done
|
||||
#13 DONE 0.1s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
Image mtusi-lab2-api Built
|
||||
Container mtusi-lab2-postgres-1 Recreate
|
||||
Container mtusi-lab2-postgres-1 Recreated
|
||||
Container mtusi-lab2-api-1 Recreate
|
||||
Container mtusi-lab2-api-1 Recreated
|
||||
Container mtusi-lab2-postgres-1 Starting
|
||||
Container mtusi-lab2-redis-1 Starting
|
||||
Container mtusi-lab2-postgres-1 Started
|
||||
Container mtusi-lab2-redis-1 Started
|
||||
Container mtusi-lab2-redis-1 Waiting
|
||||
Container mtusi-lab2-postgres-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Healthy
|
||||
Container mtusi-lab2-postgres-1 Healthy
|
||||
Container mtusi-lab2-api-1 Starting
|
||||
Container mtusi-lab2-api-1 Started
|
||||
Container mtusi-lab2-api-1 Waiting
|
||||
Container mtusi-lab2-postgres-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Healthy
|
||||
Container mtusi-lab2-postgres-1 Healthy
|
||||
Container mtusi-lab2-api-1 Healthy
|
||||
@@ -0,0 +1,116 @@
|
||||
Local configuration ready; secrets were not printed.
|
||||
Image mtusi-lab2-review-50a92603-api Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 704B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 resolving provenance for metadata file
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load build definition from Dockerfile
|
||||
#3 transferring dockerfile: 739B done
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 132B done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [internal] load build context
|
||||
#6 transferring context: 25.73kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#7 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [2/7] WORKDIR /app
|
||||
#8 CACHED
|
||||
|
||||
#9 [4/7] COPY requirements.txt ./
|
||||
#9 CACHED
|
||||
|
||||
#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt
|
||||
#10 CACHED
|
||||
|
||||
#11 [6/7] COPY src/ ./src/
|
||||
#11 CACHED
|
||||
|
||||
#12 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
#12 CACHED
|
||||
|
||||
#13 [7/7] COPY migrations/ ./migrations/
|
||||
#13 CACHED
|
||||
|
||||
#14 exporting to image
|
||||
#14 exporting layers done
|
||||
#14 exporting manifest sha256:9de8694da45da73001c72c27f49f41b2b32dda6f170fb0ce738ced563c2c9471 done
|
||||
#14 exporting config sha256:21388fc537a563dea4b9d14946bc95036d2ca3727ebed78cdeb410843e7f3700 done
|
||||
#14 exporting attestation manifest sha256:ced1e33e702d25de0036f12d77f8ba1e49335255d54c60a9de74a012000fc609 done
|
||||
#14 exporting manifest list sha256:6e728a75c92839d45b87038fbbe85f2db96e887e3614496fcc34a4ede2824c42 done
|
||||
#14 naming to docker.io/library/mtusi-lab2-review-50a92603-api:latest done
|
||||
#14 unpacking to docker.io/library/mtusi-lab2-review-50a92603-api:latest done
|
||||
#14 DONE 0.0s
|
||||
Image mtusi-lab2-review-50a92603-api Built
|
||||
Network mtusi-lab2-review-50a92603_default Creating
|
||||
Network mtusi-lab2-review-50a92603_default Created
|
||||
Volume mtusi-lab2-review-50a92603_postgres_data Creating
|
||||
Volume mtusi-lab2-review-50a92603_postgres_data Created
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Creating
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Creating
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Created
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Created
|
||||
Container mtusi-lab2-review-50a92603-api-1 Creating
|
||||
Container mtusi-lab2-review-50a92603-api-1 Created
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Starting
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Starting
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Started
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Started
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Waiting
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Waiting
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Healthy
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Healthy
|
||||
Container mtusi-lab2-review-50a92603-api-1 Starting
|
||||
Container mtusi-lab2-review-50a92603-api-1 Started
|
||||
Container mtusi-lab2-review-50a92603-api-1 Waiting
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Waiting
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Waiting
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Healthy
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Healthy
|
||||
Container mtusi-lab2-review-50a92603-api-1 Healthy
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
BASE_URL="http://127.0.0.1:18086" python3 -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_nul_round_trip (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 3.405s
|
||||
|
||||
OK
|
||||
PASS: clean local clone, fresh Compose project/volume, migrations 001+002, public smoke and all nine student tests.
|
||||
Container mtusi-lab2-review-50a92603-api-1 Stopping
|
||||
Container mtusi-lab2-review-50a92603-api-1 Stopped
|
||||
Container mtusi-lab2-review-50a92603-api-1 Removing
|
||||
Container mtusi-lab2-review-50a92603-api-1 Removed
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Stopping
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Stopping
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Stopped
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Removing
|
||||
Container mtusi-lab2-review-50a92603-redis-1 Removed
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Stopped
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Removing
|
||||
Container mtusi-lab2-review-50a92603-postgres-1 Removed
|
||||
Network mtusi-lab2-review-50a92603_default Removing
|
||||
Network mtusi-lab2-review-50a92603_default Removed
|
||||
@@ -0,0 +1,16 @@
|
||||
python3 student-tests/dependencies.py
|
||||
Container mtusi-lab2-redis-1 Stopping
|
||||
Container mtusi-lab2-redis-1 Stopped
|
||||
Container mtusi-lab2-redis-1 Starting
|
||||
Container mtusi-lab2-redis-1 Started
|
||||
Container mtusi-lab2-redis-1 Waiting
|
||||
Container mtusi-lab2-redis-1 Healthy
|
||||
Container mtusi-lab2-postgres-1 Stopping
|
||||
Container mtusi-lab2-postgres-1 Stopped
|
||||
Container mtusi-lab2-postgres-1 Starting
|
||||
Container mtusi-lab2-postgres-1 Started
|
||||
Container mtusi-lab2-postgres-1 Waiting
|
||||
Container mtusi-lab2-postgres-1 Healthy
|
||||
Container mtusi-lab2-expiry-29f426d1 Creating
|
||||
Container mtusi-lab2-expiry-29f426d1 Created
|
||||
PASS: rollback, real Redis TTL/cache, Redis outage, PostgreSQL outage/recovery and session expiry.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,21 @@
|
||||
Fresh restore run 1
|
||||
Container mtusi-lab2-api-run-2c95cff15d63 Creating
|
||||
Container mtusi-lab2-api-run-2c95cff15d63 Created
|
||||
Container mtusi-lab2-api-run-dd11ae88dd79 Creating
|
||||
Container mtusi-lab2-api-run-dd11ae88dd79 Created
|
||||
PASS: 100000 messages; two indexes; write costs; backup/restore counts, samples and full checksum match.
|
||||
Backup seconds: 0.281 Restore seconds: 0.268
|
||||
Fresh restore run 2
|
||||
Container mtusi-lab2-api-run-60ccc9d0034f Creating
|
||||
Container mtusi-lab2-api-run-60ccc9d0034f Created
|
||||
Container mtusi-lab2-api-run-2292e9ea796b Creating
|
||||
Container mtusi-lab2-api-run-2292e9ea796b Created
|
||||
PASS: 100000 messages; two indexes; write costs; backup/restore counts, samples and full checksum match.
|
||||
Backup seconds: 0.304 Restore seconds: 0.289
|
||||
Fresh restore run 3
|
||||
Container mtusi-lab2-api-run-f8267d018f9d Creating
|
||||
Container mtusi-lab2-api-run-f8267d018f9d Created
|
||||
Container mtusi-lab2-api-run-b73f035b1b8c Creating
|
||||
Container mtusi-lab2-api-run-b73f035b1b8c Created
|
||||
PASS: 100000 messages; two indexes; write costs; backup/restore counts, samples and full checksum match.
|
||||
Backup seconds: 0.28 Restore seconds: 0.267
|
||||
@@ -0,0 +1,209 @@
|
||||
[
|
||||
{
|
||||
"run": 1,
|
||||
"backup": {
|
||||
"format": "pg_dump custom",
|
||||
"bytes": 2655885,
|
||||
"dump_seconds": 0.2805722919999998,
|
||||
"bootstrap_seconds": 0.8350770000000001,
|
||||
"restore_seconds": 0.2675151250000001,
|
||||
"restored_database": "restore_8e3b0298c2c4",
|
||||
"source_container_id": "07430ea862b6",
|
||||
"restore_container_id": "db6ae0750cc8",
|
||||
"fresh_restore_volume": "mtusi-lab2-restore_8e3b0298c2c4-data",
|
||||
"source_volume": [
|
||||
"mtusi-lab2_postgres_data"
|
||||
],
|
||||
"matching_counts": true,
|
||||
"matching_all_messages_sha256": true,
|
||||
"matching_selected_messages": true,
|
||||
"restored_snapshot": {
|
||||
"counts": {
|
||||
"users": 1,
|
||||
"chats": 10,
|
||||
"chat_members": 10,
|
||||
"messages": 100000,
|
||||
"sessions": 0
|
||||
},
|
||||
"messages_sha256": "3c38a0e32acafed71651826488fb514af7371e7d43bbfeb08f73aed17d68ea4f",
|
||||
"selected_messages": [
|
||||
{
|
||||
"id": "00000000-0000-0000-0000-000000000000",
|
||||
"chat_id": "12a8a7fe-93db-49a5-a5bf-f19f24392ffd",
|
||||
"sender_id": "538d79bc-a7f0-4e96-8698-4ab577ed37c9",
|
||||
"text": "\\x41004278787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00+00:00"
|
||||
},
|
||||
{
|
||||
"id": "0000234c-4b36-4929-a3d8-31b88d3944ed",
|
||||
"chat_id": "12a8a7fe-93db-49a5-a5bf-f19f24392ffd",
|
||||
"sender_id": "538d79bc-a7f0-4e96-8698-4ab577ed37c9",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.033386+00:00"
|
||||
},
|
||||
{
|
||||
"id": "801769f9-9185-4fef-996a-754fe04331e8",
|
||||
"chat_id": "5d7a1ad8-8b31-4976-971a-eee92058b8e1",
|
||||
"sender_id": "538d79bc-a7f0-4e96-8698-4ab577ed37c9",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.002124+00:00"
|
||||
},
|
||||
{
|
||||
"id": "ffff5474-757f-457f-8bd5-894401da2910",
|
||||
"chat_id": "12a8a7fe-93db-49a5-a5bf-f19f24392ffd",
|
||||
"sender_id": "538d79bc-a7f0-4e96-8698-4ab577ed37c9",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.059892+00:00"
|
||||
},
|
||||
{
|
||||
"id": "ffffa45b-75ea-4839-a5bd-fb0a85ba0d29",
|
||||
"chat_id": "12a8a7fe-93db-49a5-a5bf-f19f24392ffd",
|
||||
"sender_id": "538d79bc-a7f0-4e96-8698-4ab577ed37c9",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.048116+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"messages": 100000,
|
||||
"nul_sample_verified": true
|
||||
},
|
||||
{
|
||||
"run": 2,
|
||||
"backup": {
|
||||
"format": "pg_dump custom",
|
||||
"bytes": 2654444,
|
||||
"dump_seconds": 0.303841,
|
||||
"bootstrap_seconds": 0.8235868330000002,
|
||||
"restore_seconds": 0.28861237500000003,
|
||||
"restored_database": "restore_dc75cd9655c0",
|
||||
"source_container_id": "07430ea862b6",
|
||||
"restore_container_id": "9a2e73441fcf",
|
||||
"fresh_restore_volume": "mtusi-lab2-restore_dc75cd9655c0-data",
|
||||
"source_volume": [
|
||||
"mtusi-lab2_postgres_data"
|
||||
],
|
||||
"matching_counts": true,
|
||||
"matching_all_messages_sha256": true,
|
||||
"matching_selected_messages": true,
|
||||
"restored_snapshot": {
|
||||
"counts": {
|
||||
"users": 1,
|
||||
"chats": 10,
|
||||
"chat_members": 10,
|
||||
"messages": 100000,
|
||||
"sessions": 0
|
||||
},
|
||||
"messages_sha256": "27b5a42196b56676462b626f38f28da114bcc7da0d4c584d4239e0286df4ab4b",
|
||||
"selected_messages": [
|
||||
{
|
||||
"id": "00000000-0000-0000-0000-000000000000",
|
||||
"chat_id": "a8efc6c4-bc38-4460-862a-01a5b31db242",
|
||||
"sender_id": "410288f9-4285-4b2c-9d5d-8ff651324300",
|
||||
"text": "\\x41004278787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00+00:00"
|
||||
},
|
||||
{
|
||||
"id": "00003164-09c1-41f2-be94-4c59dac29663",
|
||||
"chat_id": "a8efc6c4-bc38-4460-862a-01a5b31db242",
|
||||
"sender_id": "410288f9-4285-4b2c-9d5d-8ff651324300",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.048618+00:00"
|
||||
},
|
||||
{
|
||||
"id": "7f75439b-8443-451b-bd19-2fe5e0592b3b",
|
||||
"chat_id": "a8efc6c4-bc38-4460-862a-01a5b31db242",
|
||||
"sender_id": "410288f9-4285-4b2c-9d5d-8ff651324300",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.013799+00:00"
|
||||
},
|
||||
{
|
||||
"id": "ffff6737-a2c0-479f-bc93-c68bdf9ceea7",
|
||||
"chat_id": "f7079b99-bdba-46ac-a651-1c2a451fcad2",
|
||||
"sender_id": "410288f9-4285-4b2c-9d5d-8ff651324300",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.017859+00:00"
|
||||
},
|
||||
{
|
||||
"id": "ffffd4f9-50c9-4e50-a320-2b4afa0e0dec",
|
||||
"chat_id": "a8efc6c4-bc38-4460-862a-01a5b31db242",
|
||||
"sender_id": "410288f9-4285-4b2c-9d5d-8ff651324300",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.068415+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"messages": 100000,
|
||||
"nul_sample_verified": true
|
||||
},
|
||||
{
|
||||
"run": 3,
|
||||
"backup": {
|
||||
"format": "pg_dump custom",
|
||||
"bytes": 2655198,
|
||||
"dump_seconds": 0.2799740420000001,
|
||||
"bootstrap_seconds": 0.8203773750000001,
|
||||
"restore_seconds": 0.266880542,
|
||||
"restored_database": "restore_4bc6680354a1",
|
||||
"source_container_id": "07430ea862b6",
|
||||
"restore_container_id": "aff2a0068a4c",
|
||||
"fresh_restore_volume": "mtusi-lab2-restore_4bc6680354a1-data",
|
||||
"source_volume": [
|
||||
"mtusi-lab2_postgres_data"
|
||||
],
|
||||
"matching_counts": true,
|
||||
"matching_all_messages_sha256": true,
|
||||
"matching_selected_messages": true,
|
||||
"restored_snapshot": {
|
||||
"counts": {
|
||||
"users": 1,
|
||||
"chats": 10,
|
||||
"chat_members": 10,
|
||||
"messages": 100000,
|
||||
"sessions": 0
|
||||
},
|
||||
"messages_sha256": "8c3e317ab75daa8a5e97641eac51e3a349cffe2293bb83bc2ba22ce602cbf112",
|
||||
"selected_messages": [
|
||||
{
|
||||
"id": "00000000-0000-0000-0000-000000000000",
|
||||
"chat_id": "16d5e714-6c35-4ad8-8e2f-b097a33a346a",
|
||||
"sender_id": "f34b3772-5504-4f6f-861b-f22d07f10181",
|
||||
"text": "\\x41004278787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00+00:00"
|
||||
},
|
||||
{
|
||||
"id": "0000cf54-fd5c-4b78-9b83-5fdacc223d82",
|
||||
"chat_id": "16d5e714-6c35-4ad8-8e2f-b097a33a346a",
|
||||
"sender_id": "f34b3772-5504-4f6f-861b-f22d07f10181",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.039523+00:00"
|
||||
},
|
||||
{
|
||||
"id": "7fc2eae6-055d-4a9d-b3b7-4e09c7d735ea",
|
||||
"chat_id": "16d5e714-6c35-4ad8-8e2f-b097a33a346a",
|
||||
"sender_id": "f34b3772-5504-4f6f-861b-f22d07f10181",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.037757+00:00"
|
||||
},
|
||||
{
|
||||
"id": "fffe1a30-0346-4b72-b709-eb9c695c1993",
|
||||
"chat_id": "16d5e714-6c35-4ad8-8e2f-b097a33a346a",
|
||||
"sender_id": "f34b3772-5504-4f6f-861b-f22d07f10181",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.040077+00:00"
|
||||
},
|
||||
{
|
||||
"id": "fffe9bca-2d05-4455-ab60-8d0d4d519df4",
|
||||
"chat_id": "16d5e714-6c35-4ad8-8e2f-b097a33a346a",
|
||||
"sender_id": "f34b3772-5504-4f6f-861b-f22d07f10181",
|
||||
"text": "\\x78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878",
|
||||
"created_at": "2026-01-01T00:00:00.014042+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"messages": 100000,
|
||||
"nul_sample_verified": true
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
../.venv/bin/python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
../.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
../.venv/bin/python tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
BASE_URL="http://localhost:8080" ../.venv/bin/python -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract.test_concurrent_messages_and_pagination) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract.test_cursor_scope_and_validation) ... ok
|
||||
test_foreign_chat (test_api.APIContract.test_foreign_chat) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract.test_invalid_chat_creation) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract.test_malformed_json_and_extra_fields) ... ok
|
||||
test_registration_race (test_api.APIContract.test_registration_race) ... ok
|
||||
test_unicode_limits (test_api.APIContract.test_unicode_limits) ... ok
|
||||
test_unicode_nul_round_trip (test_api.APIContract.test_unicode_nul_round_trip) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract.test_unicode_password_basic_and_logout) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 2.631s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,7 @@
|
||||
Container mtusi-lab2-postgres-1 Restarting
|
||||
Container mtusi-lab2-redis-1 Restarting
|
||||
Container mtusi-lab2-api-1 Restarting
|
||||
Container mtusi-lab2-redis-1 Started
|
||||
Container mtusi-lab2-api-1 Started
|
||||
Container mtusi-lab2-postgres-1 Started
|
||||
PASS: U+0000 and literal backslash-u0000 preserved in display_name/title/text after API/PostgreSQL/Redis restart; original Bearer still valid.
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"docker": {
|
||||
"Platform": {
|
||||
"Name": "Docker Engine - Community"
|
||||
},
|
||||
"Version": "29.5.2",
|
||||
"ApiVersion": "1.54",
|
||||
"MinAPIVersion": "1.40",
|
||||
"Os": "linux",
|
||||
"Arch": "arm64",
|
||||
"Components": [
|
||||
{
|
||||
"Name": "Engine",
|
||||
"Version": "29.5.2",
|
||||
"Details": {
|
||||
"ApiVersion": "1.54",
|
||||
"Arch": "arm64",
|
||||
"BuildTime": "Wed May 20 14:39:25 2026",
|
||||
"Experimental": "false",
|
||||
"GitCommit": "568f755",
|
||||
"GoVersion": "go1.26.3",
|
||||
"KernelVersion": "6.8.0-117-generic",
|
||||
"MinAPIVersion": "1.40",
|
||||
"Module": "github.com/moby/moby/v2",
|
||||
"ModuleVersion": "v2.0.0+unknown",
|
||||
"Os": "linux"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "containerd",
|
||||
"Version": "v2.2.4",
|
||||
"Details": {
|
||||
"GitCommit": "193637f7ee8ae5f5aa5248f49e7baa3e6164966e"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "runc",
|
||||
"Version": "1.3.5",
|
||||
"Details": {
|
||||
"GitCommit": "v1.3.5-0-g488fc13e"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "docker-init",
|
||||
"Version": "0.19.0",
|
||||
"Details": {
|
||||
"GitCommit": "de40ad0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"GitCommit": "568f755",
|
||||
"GoVersion": "go1.26.3",
|
||||
"KernelVersion": "6.8.0-117-generic",
|
||||
"BuildTime": "2026-05-20T14:39:25.000000000+00:00"
|
||||
},
|
||||
"postgres_settings": {
|
||||
"fsync": "on",
|
||||
"synchronous_commit": "on",
|
||||
"full_page_writes": "on",
|
||||
"migration_versions": [
|
||||
"001_initial.sql"
|
||||
]
|
||||
},
|
||||
"api_python": "Python 3.12.8",
|
||||
"containers": [
|
||||
{
|
||||
"service": "api",
|
||||
"user": "10001:10001",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"oom_killed": false,
|
||||
"health": "healthy"
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"user": "",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"oom_killed": false,
|
||||
"health": "healthy"
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"user": "",
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false,
|
||||
"health": "healthy"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,13 @@
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 8 tests in 11.051s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,2 @@
|
||||
../.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,42 @@
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY
|
||||
, username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$')
|
||||
, display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
, password_hash TEXT NOT NULL
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
;
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY
|
||||
, title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, last_message_at TIMESTAMPTZ
|
||||
)
|
||||
;
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, user_id UUID NOT NULL REFERENCES users(id)
|
||||
, PRIMARY KEY (chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX chat_members_user ON chat_members (user_id, chat_id)
|
||||
;
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, sender_id UUID NOT NULL REFERENCES users(id)
|
||||
, text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
, created_at TIMESTAMPTZ NOT NULL
|
||||
, FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id)
|
||||
;
|
||||
CREATE TABLE sessions (
|
||||
token_hash TEXT PRIMARY KEY
|
||||
, user_id UUID NOT NULL REFERENCES users(id)
|
||||
, expires_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
;
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at)
|
||||
;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA.
|
||||
-- API validation keeps the contract's code-point limits; these bounds cap bytes.
|
||||
ALTER TABLE users
|
||||
DROP CONSTRAINT users_display_name_check
|
||||
, ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8')
|
||||
, ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
|
||||
;
|
||||
ALTER TABLE chats
|
||||
DROP CONSTRAINT chats_title_check
|
||||
, ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8')
|
||||
, ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
|
||||
;
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_check
|
||||
, ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8')
|
||||
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
|
||||
;
|
||||
@@ -0,0 +1,4 @@
|
||||
psycopg[binary]==3.2.3
|
||||
psycopg-pool==3.2.4
|
||||
redis==5.2.1
|
||||
typing-extensions==4.16.0
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local lab credentials once; never print values or overwrite an existing setup."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / 'secrets'
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
password_path = directory / 'postgres_password'
|
||||
if not password_path.exists():
|
||||
password_path.write_text(secrets.token_hex(24))
|
||||
password_path.chmod(0o600)
|
||||
password = password_path.read_text().strip()
|
||||
for name, value in {
|
||||
'database_url': 'postgresql://messenger:' + password + '@postgres:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Docker Compose bind-mounted secrets must be readable by UID 10001.
|
||||
# The containing host directory is accessible only to its owner.
|
||||
path.chmod(0o644)
|
||||
env = root / '.env'
|
||||
if not env.exists():
|
||||
env.write_text((root / '.env.example').read_text())
|
||||
env.chmod(0o600)
|
||||
print('Local configuration ready; secrets were not printed.')
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
python3 scripts/snapshot.py > evidence/restart-before.json
|
||||
docker compose restart postgres redis
|
||||
docker compose up -d --wait postgres redis
|
||||
docker compose restart api
|
||||
docker compose up -d --wait api
|
||||
python3 scripts/snapshot.py > evidence/restart-after.json
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Record restart evidence without env, credentials or machine-specific labels."""
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', 'api', 'postgres', 'redis'], text=True).split()
|
||||
containers = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True))
|
||||
result = []
|
||||
for container in containers:
|
||||
result.append(dict(service=container['Config']['Labels']['com.docker.compose.service'], container_id=container['Id'][:12], started_at=container['State']['StartedAt'], status=container['State']['Status'], volumes=[m['Name'] for m in container['Mounts'] if m['Type'] == 'volume']))
|
||||
print(json.dumps(sorted(result, key=lambda x: x['service']), indent=2))
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Shared HTTP contract; storage is supplied by each laboratory."""
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def invalid():
|
||||
raise APIError(400, 'invalid_request', 'Неверные параметры запроса')
|
||||
|
||||
|
||||
def unauthorized():
|
||||
raise APIError(401, 'unauthorized', 'Требуется действительная идентификация')
|
||||
|
||||
|
||||
def missing():
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
|
||||
|
||||
def identifier(value):
|
||||
try:
|
||||
if not isinstance(value, str):
|
||||
invalid()
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
invalid()
|
||||
|
||||
|
||||
def text(value, maximum, minimum=1):
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def fields(value, required):
|
||||
if not isinstance(value, dict) or set(value) != set(required):
|
||||
invalid()
|
||||
|
||||
|
||||
def username(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def now():
|
||||
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def page_args(query, scope):
|
||||
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
|
||||
invalid()
|
||||
limit = query.get('limit', ['50'])[0]
|
||||
if not re.fullmatch(r'[0-9]+', limit):
|
||||
invalid()
|
||||
try:
|
||||
limit = int(limit)
|
||||
except ValueError:
|
||||
invalid()
|
||||
if not 1 <= limit <= 100:
|
||||
invalid()
|
||||
cursor = None
|
||||
if 'cursor' in query:
|
||||
raw = query['cursor'][0]
|
||||
try:
|
||||
if not raw or len(raw) > 1024:
|
||||
invalid()
|
||||
decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope:
|
||||
invalid()
|
||||
stamp = decoded[1]
|
||||
if not isinstance(stamp, str) or not stamp.endswith('Z'):
|
||||
invalid()
|
||||
stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
cursor = (stamp, identifier(decoded[2]))
|
||||
except (ValueError, TypeError, UnicodeError):
|
||||
invalid()
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def page(items, limit, scope):
|
||||
more = len(items) > limit
|
||||
items = items[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = items[-1]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
|
||||
return {'items': items, 'next_cursor': cursor}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
import base64
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
import redis
|
||||
|
||||
from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username
|
||||
|
||||
KDF_SLOTS = threading.BoundedSemaphore(2)
|
||||
|
||||
|
||||
def setting(name):
|
||||
path = os.environ.get(name + '_FILE')
|
||||
return Path(path).read_text().strip() if path else os.environ[name]
|
||||
|
||||
|
||||
def public(row):
|
||||
return {k: (v.decode('utf-8') if isinstance(v, bytes) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'}
|
||||
|
||||
|
||||
def password_hash(password, salt=None):
|
||||
salt = salt or secrets.token_bytes(16)
|
||||
if not KDF_SLOTS.acquire(timeout=1.5):
|
||||
raise APIError(503, 'unavailable', 'Аутентификация временно перегружена')
|
||||
try:
|
||||
digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
|
||||
finally:
|
||||
KDF_SLOTS.release()
|
||||
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
|
||||
|
||||
|
||||
class Store:
|
||||
persistent = True
|
||||
|
||||
def __init__(self):
|
||||
self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
|
||||
if not 1 <= self.ttl <= 86400:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be 1..86400')
|
||||
self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=8, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True)
|
||||
self.pool.wait(timeout=10)
|
||||
self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True)
|
||||
self.dummy_hash = password_hash('dummy-password-for-unknown-users')
|
||||
self.migrate()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
|
||||
|
||||
def migrate(self):
|
||||
with self.connection() as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2301002)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)')
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||
if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone():
|
||||
conn.execute(path.read_text())
|
||||
conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,))
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
|
||||
def ready(self):
|
||||
with self.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
|
||||
def register(self, body):
|
||||
fields(body, ('username', 'display_name', 'password'))
|
||||
name, display = username(body['username']), text(body['display_name'], 100)
|
||||
# Passwords may contain spaces and colons; only length/UTF-8 are restricted.
|
||||
password = body['password']
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||
invalid()
|
||||
try:
|
||||
password.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
hashed = password_hash(password)
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('''
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone()
|
||||
return public(row)
|
||||
|
||||
def login(self, headers):
|
||||
authorization = headers.get('Authorization', '')
|
||||
try:
|
||||
scheme, encoded = authorization.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
unauthorized()
|
||||
name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
except (ValueError, UnicodeError):
|
||||
unauthorized()
|
||||
if not re_username(name) or not 12 <= len(password) <= 128:
|
||||
unauthorized()
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone()
|
||||
hashed = row['password_hash'] if row else self.dummy_hash
|
||||
verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed)
|
||||
if row is None or not verified:
|
||||
unauthorized()
|
||||
token = secrets.token_urlsafe(32)
|
||||
with self.connection() as conn:
|
||||
expiry = conn.execute('''
|
||||
INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second')
|
||||
RETURNING expires_at
|
||||
''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at']
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)}
|
||||
|
||||
def token_hash(self, headers):
|
||||
parts = headers.get('Authorization', '').split(' ')
|
||||
if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii():
|
||||
unauthorized()
|
||||
return hashlib.sha256(parts[1].encode()).hexdigest()
|
||||
|
||||
def authenticate(self, headers):
|
||||
token_hash = self.token_hash(headers)
|
||||
with self.connection() as conn:
|
||||
session = conn.execute('''
|
||||
SELECT user_id
|
||||
FROM sessions
|
||||
WHERE 1=1
|
||||
AND token_hash = %s
|
||||
AND expires_at > clock_timestamp()
|
||||
''', (token_hash,)).fetchone()
|
||||
if session is None:
|
||||
unauthorized()
|
||||
key = 'user:' + str(session['user_id'])
|
||||
try:
|
||||
cached = self.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError):
|
||||
pass
|
||||
row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone()
|
||||
if row is None:
|
||||
unauthorized()
|
||||
user = public(row)
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
pass
|
||||
return user
|
||||
|
||||
def logout(self, headers):
|
||||
with self.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),))
|
||||
|
||||
def chat_in(self, conn, actor, chat_id):
|
||||
row = conn.execute('''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE 1=1
|
||||
AND c.id = %s
|
||||
AND cm.user_id = %s
|
||||
''', (chat_id, actor['id'])).fetchone()
|
||||
if row is None:
|
||||
missing()
|
||||
members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall()
|
||||
return dict(public(row), member_ids=[str(v['user_id']) for v in members])
|
||||
|
||||
def chat(self, actor, chat_id):
|
||||
with self.connection() as conn:
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
|
||||
def create_chat(self, actor, body):
|
||||
fields(body, ('title', 'member_ids'))
|
||||
title = text(body['title'], 100)
|
||||
members = body['member_ids']
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
invalid()
|
||||
members = [identifier(v) for v in members]
|
||||
if len(set(members)) != len(members) or actor['id'] not in members:
|
||||
invalid()
|
||||
with self.connection() as conn:
|
||||
chat_id = uuid.uuid4()
|
||||
conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8')))
|
||||
# Checking after INSERT exercises rollback of the whole composite operation.
|
||||
found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall()
|
||||
if len(found) != len(members):
|
||||
invalid()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members])
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
|
||||
def list_chats(self, actor, limit, cursor, scope):
|
||||
with self.connection() as conn:
|
||||
where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else ''
|
||||
params = [actor['id']] + list(cursor or ()) + [limit + 1]
|
||||
rows = conn.execute('''
|
||||
SELECT c.id
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE 1=1
|
||||
AND cm.user_id = %s
|
||||
''' + where + '''
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
''', params).fetchall()
|
||||
return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope)
|
||||
|
||||
def send(self, actor, chat_id, body):
|
||||
fields(body, ('text',))
|
||||
content = text(body['text'], 4000)
|
||||
with self.connection() as conn:
|
||||
self.chat_in(conn, actor, chat_id)
|
||||
# Row locking serializes sends per chat, including the timestamp assignment.
|
||||
stamp = conn.execute('''
|
||||
UPDATE chats
|
||||
SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond')
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING last_message_at
|
||||
''', (chat_id,)).fetchone()['last_message_at']
|
||||
row = conn.execute('''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp)).fetchone()
|
||||
return public(row)
|
||||
|
||||
def history(self, actor, chat_id, limit, cursor, scope):
|
||||
with self.connection() as conn:
|
||||
self.chat_in(conn, actor, chat_id)
|
||||
where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else ''
|
||||
params = [chat_id] + list(cursor or ()) + [limit + 1]
|
||||
rows = conn.execute('''
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE 1=1
|
||||
AND chat_id = %s
|
||||
''' + where + '''
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
''', params).fetchall()
|
||||
return page([public(v) for v in rows], limit, scope)
|
||||
|
||||
|
||||
def re_username(value):
|
||||
import re
|
||||
return re.fullmatch(r'[a-z0-9_]{3,32}', value)
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
"""Bounded threaded HTTP server, stdlib only in lab1."""
|
||||
import concurrent.futures
|
||||
import http.server
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from common import APIError, identifier, invalid, missing, page_args
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
invalid()
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(3)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def reply(self, status, body):
|
||||
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(raw)))
|
||||
self.send_header('X-Request-Id', self.request_id)
|
||||
if status == 401 and self.server.store.persistent:
|
||||
self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def body(self):
|
||||
lengths = self.headers.get_all('Content-Length', [])
|
||||
if len(lengths) != 1 or self.headers.get('Transfer-Encoding'):
|
||||
invalid()
|
||||
try:
|
||||
length = int(lengths[0])
|
||||
except ValueError:
|
||||
invalid()
|
||||
if length > 32768:
|
||||
raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт')
|
||||
if length <= 0:
|
||||
invalid()
|
||||
if self.headers.get_content_type() != 'application/json':
|
||||
raise APIError(415, 'unsupported_media_type', 'Требуется application/json')
|
||||
try:
|
||||
raw = self.rfile.read(length)
|
||||
if len(raw) != length:
|
||||
invalid()
|
||||
return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid())
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
invalid()
|
||||
|
||||
def dispatch(self):
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.route = urlsplit(self.path).path
|
||||
status = 500
|
||||
try:
|
||||
status, result = self.route_request()
|
||||
except APIError as exc:
|
||||
status = exc.status
|
||||
result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
|
||||
except Exception:
|
||||
# No bodies, credentials, tokens, SQL or exception details in logs/responses.
|
||||
LOG.error('request_failed request_id=%s', self.request_id)
|
||||
result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
|
||||
try:
|
||||
self.reply(status, result)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
||||
pass
|
||||
LOG.info('method=%s status=%s request_id=%s', self.command, status, self.request_id)
|
||||
|
||||
def route_request(self):
|
||||
method, path, store = self.command, self.route, self.server.store
|
||||
if method == 'GET' and path in ('/health/live', '/health/ready'):
|
||||
if path.endswith('ready'):
|
||||
store.ready()
|
||||
return 200, {'status': 'ok'}
|
||||
if method == 'POST' and path == '/api/v1/users':
|
||||
return 201, store.register(self.body())
|
||||
if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions':
|
||||
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
||||
invalid()
|
||||
return 201, store.login(self.headers)
|
||||
actor = store.authenticate(self.headers)
|
||||
if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current':
|
||||
store.logout(self.headers)
|
||||
return 204, None
|
||||
if method == 'GET' and path == '/api/v1/users/me':
|
||||
return 200, actor
|
||||
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
|
||||
if path == '/api/v1/chats':
|
||||
if method == 'POST':
|
||||
return 201, store.create_chat(actor, self.body())
|
||||
if method == 'GET':
|
||||
scope = 'chats:' + actor['id']
|
||||
limit, cursor = page_args(query, scope)
|
||||
return 200, store.list_chats(actor, limit, cursor, scope)
|
||||
parts = path.split('/')
|
||||
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']:
|
||||
chat_id = identifier(parts[4])
|
||||
store.chat(actor, chat_id)
|
||||
if len(parts) == 5 and method == 'GET':
|
||||
return 200, store.chat(actor, chat_id)
|
||||
if len(parts) == 6 and parts[5] == 'messages':
|
||||
if method == 'POST':
|
||||
return 201, store.send(actor, chat_id, self.body())
|
||||
if method == 'GET':
|
||||
scope = 'messages:' + chat_id
|
||||
limit, cursor = page_args(query, scope)
|
||||
return 200, store.history(actor, chat_id, limit, cursor, scope)
|
||||
missing()
|
||||
|
||||
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch
|
||||
|
||||
|
||||
class Server(http.server.HTTPServer):
|
||||
request_queue_size = 64
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, address, store):
|
||||
self.store = store
|
||||
workers = int(os.environ.get('HTTP_WORKERS', '16'))
|
||||
if not 1 <= workers <= 128:
|
||||
raise ValueError('HTTP_WORKERS must be 1..128')
|
||||
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
|
||||
self.slots = threading.BoundedSemaphore(workers * 2)
|
||||
self.connections = set()
|
||||
self.connections_lock = threading.Lock()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def process_request(self, request, address):
|
||||
if not self.slots.acquire(blocking=False):
|
||||
rid = uuid.uuid4().hex
|
||||
raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode()
|
||||
try:
|
||||
request.settimeout(0.2)
|
||||
request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw)
|
||||
except OSError:
|
||||
pass
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
with self.connections_lock:
|
||||
self.connections.add(request)
|
||||
self.executor.submit(self.worker, request, address)
|
||||
|
||||
def worker(self, request, address):
|
||||
try:
|
||||
self.finish_request(request, address)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
with self.connections_lock:
|
||||
self.connections.discard(request)
|
||||
self.slots.release()
|
||||
|
||||
def close_connections(self):
|
||||
with self.connections_lock:
|
||||
connections = list(self.connections)
|
||||
for connection in connections:
|
||||
try:
|
||||
connection.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def server_close(self):
|
||||
super().server_close()
|
||||
# Bound draining even when a slow client keeps an incomplete request alive.
|
||||
deadline = threading.Timer(5, self.close_connections)
|
||||
deadline.start()
|
||||
try:
|
||||
self.executor.shutdown(wait=True)
|
||||
finally:
|
||||
deadline.cancel()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s')
|
||||
from postgres import Store
|
||||
store = Store()
|
||||
server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store)
|
||||
stopping = threading.Event()
|
||||
|
||||
def stop(signum, frame):
|
||||
if not stopping.is_set():
|
||||
stopping.set()
|
||||
LOG.info('shutdown_started signal=%s', signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
LOG.info('started port=%s', server.server_address[1])
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
store.close()
|
||||
LOG.info('shutdown_complete')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Destructive only to this repository's disposable lab services; volumes preserved."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
os.chdir(ROOT)
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(['docker', 'compose', *args], text=True).strip()
|
||||
|
||||
|
||||
def wait_ready(client):
|
||||
deadline = time.monotonic() + 45
|
||||
while True:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(timeout=5)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
result = {}
|
||||
query = "SELECT count(*) FROM chats;"
|
||||
before = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
|
||||
c.call('POST', '/api/v1/chats', 400, actor=alice, body={'title': 'must rollback', 'member_ids': [alice['id'], str(uuid.uuid4())]})
|
||||
after = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
|
||||
assert before == after, 'failed chat left an orphan'
|
||||
result['composite_operation_rollback'] = True
|
||||
# Force a miss and observe the actual user cache and TTL, without printing its values.
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'DEL', 'user:' + alice['id'])
|
||||
c.call('GET', '/api/v1/users/me', actor=alice)
|
||||
ttl = int(compose('exec', '-T', 'redis', 'redis-cli', 'TTL', 'user:' + alice['id']))
|
||||
assert 0 < ttl <= 60
|
||||
result['redis_user_cache_ttl_seconds'] = ttl
|
||||
compose('stop', 'redis')
|
||||
try:
|
||||
started = time.monotonic()
|
||||
c.call('GET', '/api/v1/users/me', actor=alice)
|
||||
c.message(alice, chat, 'Redis unavailable; PostgreSQL remains authoritative')
|
||||
c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Bearer ' + 'x' * 43})
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
result['redis_outage_seconds'] = time.monotonic() - started
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 'redis')
|
||||
c.call('GET', '/api/v1/users/me', actor=alice)
|
||||
result['redis_recovered_without_relogin'] = True
|
||||
compose('stop', 'postgres')
|
||||
try:
|
||||
started = time.monotonic()
|
||||
c.call('GET', '/api/v1/users/me', 503, actor=alice)
|
||||
result['postgres_outage_response_seconds'] = time.monotonic() - started
|
||||
assert result['postgres_outage_response_seconds'] < 4
|
||||
c.call('GET', '/health/ready', 503)
|
||||
c.call('GET', '/health/live', schema='Health')
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 'postgres')
|
||||
wait_ready(c)
|
||||
c.call('GET', '/api/v1/users/me', actor=alice)
|
||||
result['postgres_recovered_without_relogin'] = True
|
||||
# A separate short-TTL API uses the same authoritative PostgreSQL without changing the main API.
|
||||
name = 'mtusi-lab2-expiry-' + uuid.uuid4().hex[:8]
|
||||
compose('run', '-d', '--no-deps', '--name', name, '-e', 'SESSION_TTL_SECONDS=2', '-p', '127.0.0.1:18083:8080', 'api')
|
||||
try:
|
||||
short = Client('http://127.0.0.1:18083')
|
||||
wait_ready(short)
|
||||
actor = short.user('expiry')
|
||||
short.call('GET', '/api/v1/users/me', actor=actor)
|
||||
time.sleep(2.2)
|
||||
short.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
result['short_ttl_expired'] = True
|
||||
finally:
|
||||
subprocess.run(['docker', 'rm', '-f', name], check=True, stdout=subprocess.DEVNULL)
|
||||
(ROOT / 'evidence/dependencies.json').write_text(json.dumps(result, indent=2))
|
||||
print('PASS: rollback, real Redis TTL/cache, Redis outage, PostgreSQL outage/recovery and session expiry.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Runs inside the API image, against a separate synthetic research database."""
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'src'))
|
||||
from postgres import setting
|
||||
|
||||
|
||||
def snapshot(conn):
|
||||
counts = {}
|
||||
for table in ('users', 'chats', 'chat_members', 'messages', 'sessions'):
|
||||
counts[table] = conn.execute(sql.SQL('SELECT count(*) FROM {}').format(sql.Identifier(table))).fetchone()[0]
|
||||
digest = hashlib.sha256()
|
||||
selected = []
|
||||
with conn.cursor(name='checksum') as cur:
|
||||
cur.execute('SELECT row_to_json(m)::text FROM messages m ORDER BY id')
|
||||
for i, (row,) in enumerate(cur):
|
||||
digest.update(row.encode() + b'\n')
|
||||
if i in (0, 1, 50000, 99998, 99999):
|
||||
selected.append(json.loads(row))
|
||||
return dict(counts=counts, messages_sha256=digest.hexdigest(), selected_messages=selected)
|
||||
|
||||
|
||||
def main():
|
||||
url = setting('DATABASE_URL')
|
||||
if '--verify' in sys.argv:
|
||||
with psycopg.connect(make_conninfo(url, dbname=os.environ['RESEARCH_DATABASE'], host=os.environ['RESEARCH_HOST'])) as conn:
|
||||
print(json.dumps(snapshot(conn)))
|
||||
return
|
||||
database = 'research_' + uuid.uuid4().hex[:12]
|
||||
with psycopg.connect(url, autocommit=True) as admin:
|
||||
admin.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(database)))
|
||||
conn = psycopg.connect(make_conninfo(url, dbname=database))
|
||||
for migration in sorted((ROOT / 'migrations').glob('*.sql')):
|
||||
conn.execute(migration.read_text())
|
||||
user = uuid.uuid4()
|
||||
conn.execute('INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s)', (user, 'research_user', b'Synthetic', 'not-an-authenticatable-password'))
|
||||
sizes = [70000, 20000, 5000, 2000, 1000, 800, 500, 300, 200, 200]
|
||||
chats = [uuid.uuid4() for _ in sizes]
|
||||
base = dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc)
|
||||
for chat in chats:
|
||||
conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat, b'Synthetic'))
|
||||
conn.execute('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', (chat, user))
|
||||
with conn.cursor().copy('COPY messages (id, chat_id, sender_id, text, created_at) FROM STDIN') as copy:
|
||||
for chat, count in zip(chats, sizes):
|
||||
for i in range(count):
|
||||
first = chat == chats[0] and i == 0
|
||||
mid = uuid.UUID(int=0) if first else uuid.uuid4()
|
||||
content = b'A\x00B' + b'x' * 61 if first else b'x' * 64
|
||||
copy.write_row((mid, chat, user, content, base + dt.timedelta(microseconds=i)))
|
||||
conn.commit()
|
||||
results = []
|
||||
query = '''
|
||||
SELECT
|
||||
id
|
||||
, chat_id
|
||||
, sender_id
|
||||
, text
|
||||
, created_at
|
||||
FROM messages
|
||||
WHERE 1=1
|
||||
AND chat_id = %s
|
||||
AND (created_at, id) > (%s, %s)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51
|
||||
'''
|
||||
for variant, columns in (('chat_only', 'chat_id'), ('composite', 'chat_id, created_at, id')):
|
||||
conn.execute('DROP INDEX messages_history')
|
||||
# The two SQL fragments are constants, never request input.
|
||||
conn.execute('CREATE INDEX messages_history ON messages (' + columns + ')')
|
||||
conn.execute('ANALYZE messages')
|
||||
conn.commit()
|
||||
reads = []
|
||||
for position in (0, 1, 9):
|
||||
params = (chats[position], base + dt.timedelta(microseconds=sizes[position] * 4 // 5), uuid.UUID(int=0))
|
||||
conn.execute(query, params).fetchall() # Warm the same page before both variants.
|
||||
plans = [conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + query, params).fetchone()[0][0] for _ in range(5)]
|
||||
reads.append(dict(chat_messages=sizes[position], median_execution_ms=statistics.median(v['Execution Time'] for v in plans), plans=plans))
|
||||
conn.commit()
|
||||
writes = []
|
||||
for repeat in range(3):
|
||||
ids = [uuid.uuid4() for _ in range(1000)]
|
||||
before = conn.execute('SELECT pg_current_wal_insert_lsn()').fetchone()[0]
|
||||
conn.commit()
|
||||
started = time.perf_counter()
|
||||
with conn.cursor().copy('COPY messages (id, chat_id, sender_id, text, created_at) FROM STDIN') as copy:
|
||||
for i, mid in enumerate(ids):
|
||||
copy.write_row((mid, chats[0], user, b'write-cost', base + dt.timedelta(seconds=1, microseconds=i)))
|
||||
conn.commit()
|
||||
elapsed = (time.perf_counter() - started) * 1000
|
||||
wal = conn.execute('SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(), %s::pg_lsn)', (before,)).fetchone()[0]
|
||||
writes.append(dict(rows=1000, commit_ms=elapsed, wal_bytes=int(wal)))
|
||||
conn.execute('DELETE FROM messages WHERE 1=1 AND id = ANY(%s::uuid[])', (ids,))
|
||||
conn.commit()
|
||||
index_size = conn.execute("SELECT pg_relation_size('messages_history')").fetchone()[0]
|
||||
results.append(dict(variant=variant, index_bytes=index_size, reads=reads, writes=writes))
|
||||
captured = snapshot(conn)
|
||||
assert captured['counts']['messages'] == 100000
|
||||
assert captured['selected_messages'][0]['text'] == '\\x' + (b'A\x00B' + b'x' * 61).hex()
|
||||
version = conn.execute('SELECT version()').fetchone()[0]
|
||||
conn.close()
|
||||
print(json.dumps(dict(database=database, postgresql=version, sizes=sizes, query=query, repeats=5, results=results, snapshot=captured)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Index study, pg_dump, restore into a new database, full message checksum."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
os.chdir(ROOT)
|
||||
COMPOSE = ['docker', 'compose']
|
||||
RUN = COMPOSE + ['run', '--rm', '--no-deps', '-T', '-v', str(ROOT / 'student-tests') + ':/app/student-tests:ro']
|
||||
|
||||
result = json.loads(subprocess.check_output(RUN + ['api', 'python', 'student-tests/index_study.py'], text=True))
|
||||
source = result['database']
|
||||
restore = 'restore_' + uuid.uuid4().hex[:12]
|
||||
restore_container = 'mtusi-lab2-' + restore
|
||||
restore_volume = restore_container + '-data'
|
||||
directory = ROOT / 'backups'
|
||||
directory.mkdir(exist_ok=True, mode=0o700)
|
||||
backup = directory / (source + '.dump')
|
||||
started = time.perf_counter()
|
||||
with backup.open('wb') as output:
|
||||
subprocess.run(COMPOSE + ['exec', '-T', 'postgres', 'pg_dump', '-U', 'messenger', '-Fc', source], stdout=output, check=True)
|
||||
backup.chmod(0o600)
|
||||
dump_seconds = time.perf_counter() - started
|
||||
source_id = subprocess.check_output(COMPOSE + ['ps', '-q', 'postgres'], text=True).strip()
|
||||
source_info = json.loads(subprocess.check_output(['docker', 'inspect', source_id], text=True))[0]
|
||||
network = next(iter(source_info['NetworkSettings']['Networks']))
|
||||
image = source_info['Config']['Image']
|
||||
subprocess.run(['docker', 'volume', 'create', restore_volume], check=True, stdout=subprocess.DEVNULL)
|
||||
started_bootstrap = time.perf_counter()
|
||||
restore_id = subprocess.check_output(['docker', 'run', '-d', '--name', restore_container, '--network', network, '--cpus=1', '--memory=512m', '-e', 'POSTGRES_USER=messenger', '-e', 'POSTGRES_DB=' + restore, '-e', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password', '--mount', 'type=volume,src=' + restore_volume + ',dst=/var/lib/postgresql/data', '--mount', 'type=bind,src=' + str(ROOT / 'secrets/postgres_password') + ',dst=/run/secrets/postgres_password,readonly', image], text=True).strip()
|
||||
try:
|
||||
deadline = time.monotonic() + 45
|
||||
# The entrypoint's temporary init server accepts Unix sockets only.
|
||||
# TCP becomes ready only after initialization has finished and it is replaced.
|
||||
while subprocess.run(['docker', 'exec', restore_container, 'pg_isready', '-h', '127.0.0.1', '-U', 'messenger', '-d', restore], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:
|
||||
assert time.monotonic() < deadline, 'new PostgreSQL did not become ready'
|
||||
time.sleep(0.3)
|
||||
bootstrap_seconds = time.perf_counter() - started_bootstrap
|
||||
started = time.perf_counter()
|
||||
with backup.open('rb') as data:
|
||||
subprocess.run(['docker', 'exec', '-i', restore_container, 'pg_restore', '-h', '127.0.0.1', '-U', 'messenger', '--no-owner', '--exit-on-error', '-d', restore], stdin=data, check=True)
|
||||
restore_seconds = time.perf_counter() - started
|
||||
verified = json.loads(subprocess.check_output(RUN + ['-e', 'RESEARCH_DATABASE=' + restore, '-e', 'RESEARCH_HOST=' + restore_container, 'api', 'python', 'student-tests/index_study.py', '--verify'], text=True))
|
||||
finally:
|
||||
subprocess.run(['docker', 'stop', '-t', '10', restore_container], check=True, stdout=subprocess.DEVNULL)
|
||||
assert result['snapshot'] == verified, 'restored contents differ'
|
||||
result['backup'] = dict(format='pg_dump custom', bytes=backup.stat().st_size, dump_seconds=dump_seconds, bootstrap_seconds=bootstrap_seconds, restore_seconds=restore_seconds, restored_database=restore, source_container_id=source_id[:12], restore_container_id=restore_id[:12], fresh_restore_volume=restore_volume, source_volume=[m['Name'] for m in source_info['Mounts'] if m['Type'] == 'volume'], matching_counts=True, matching_all_messages_sha256=True, matching_selected_messages=True, restored_snapshot=verified)
|
||||
(ROOT / 'evidence/index-study.json').write_text(json.dumps(result, indent=2))
|
||||
print('PASS: 100000 messages; two indexes; write costs; backup/restore counts, samples and full checksum match.')
|
||||
print('Backup seconds:', round(dump_seconds, 3), 'Restore seconds:', round(restore_seconds, 3))
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Extra black-box tests, without changing the course's public tests."""
|
||||
import concurrent.futures
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, LAB
|
||||
|
||||
|
||||
class APIContract(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.client = Client()
|
||||
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
|
||||
self.path = '/api/v1/chats/' + self.chat['id']
|
||||
|
||||
def test_malformed_json_and_extra_fields(self):
|
||||
for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'):
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
|
||||
for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}):
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body)
|
||||
deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}'
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'})
|
||||
|
||||
def test_unicode_limits(self):
|
||||
result = self.client.message(self.alice, self.chat, '👩' * 4000)['json']
|
||||
self.assertEqual(result['text'], '👩' * 4000)
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001})
|
||||
|
||||
def test_unicode_nul_round_trip(self):
|
||||
import base64
|
||||
value = 'A\x00B\\u0000👩'
|
||||
registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value)
|
||||
user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
|
||||
self.assertEqual(user['display_name'], value)
|
||||
credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
|
||||
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
|
||||
self.assertEqual(session['user'], user)
|
||||
actor = {'id': user['id'], 'token': session['token']}
|
||||
# Exercise the database path and then the Redis profile cache.
|
||||
for _ in range(2):
|
||||
self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user)
|
||||
chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json']
|
||||
self.assertEqual(chat['title'], value)
|
||||
self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat)
|
||||
contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00']
|
||||
for content in contents:
|
||||
self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content)
|
||||
history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json']
|
||||
self.assertEqual([message['text'] for message in history['items']], contents)
|
||||
|
||||
def test_foreign_chat(self):
|
||||
for suffix in ('', '/messages'):
|
||||
self.client.call('GET', self.path + suffix, 404, actor=self.eve)
|
||||
self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'})
|
||||
self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice)
|
||||
|
||||
def test_registration_race(self):
|
||||
body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20])
|
||||
def register(_):
|
||||
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
statuses = list(executor.map(register, range(10)))
|
||||
self.assertEqual(statuses.count(201), 1)
|
||||
self.assertEqual(statuses.count(409), 9)
|
||||
|
||||
def test_invalid_chat_creation(self):
|
||||
members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True])
|
||||
for member_ids in members:
|
||||
self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids})
|
||||
|
||||
def test_cursor_scope_and_validation(self):
|
||||
for i in range(3):
|
||||
self.client.message(self.alice, self.chat, str(i))
|
||||
cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor']
|
||||
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
|
||||
self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice)
|
||||
for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'):
|
||||
self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice)
|
||||
|
||||
def test_concurrent_messages_and_pagination(self):
|
||||
def send(i):
|
||||
return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json']
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
sent = list(executor.map(send, range(100)))
|
||||
found = []
|
||||
cursor = None
|
||||
while True:
|
||||
url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '')
|
||||
page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json']
|
||||
found.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
break
|
||||
self.assertEqual(len(found), 100)
|
||||
self.assertEqual(len({v['id'] for v in found}), 100)
|
||||
self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent})
|
||||
self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id'])))
|
||||
|
||||
@unittest.skipUnless(LAB >= 2, 'sessions start in lab2')
|
||||
def test_unicode_password_basic_and_logout(self):
|
||||
import base64
|
||||
registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'}
|
||||
self.client.call('POST', '/api/v1/users', 201, body=registration)
|
||||
encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
|
||||
headers = {'Authorization': 'Basic ' + encoded}
|
||||
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json']
|
||||
actor = {'id': session['user']['id'], 'token': session['token']}
|
||||
self.client.call('GET', '/api/v1/users/me', actor=actor)
|
||||
self.client.call('GET', '/api/v1/users/me', 401, headers=headers)
|
||||
for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'):
|
||||
self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization})
|
||||
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user