forked from mtusi2026/mtusi2026lab2
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef96551b7d |
@@ -17,4 +17,5 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
output
|
||||
.DS_Store
|
||||
|
||||
+2
-1
@@ -7,5 +7,6 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
CURSOR_KEY_FILE=/run/secrets/cursor_key
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
* text=auto
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.conf text eol=lf
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Lab 2 application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Local synthetic secrets and stack
|
||||
run: |
|
||||
python scripts/bootstrap.py
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
- name: Grade 4 checks
|
||||
run: make verify PYTHON=python
|
||||
- name: Evidence without secrets
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: lab2-evidence
|
||||
path: evidence/
|
||||
- name: Stop stack, preserve volumes
|
||||
if: always()
|
||||
run: docker compose down
|
||||
@@ -28,5 +28,10 @@ coverage/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
output/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
# Personal study notes stay local
|
||||
DEFENSE.md
|
||||
DEFENSE_STEPS.md
|
||||
MR_DRAFT.md
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
FROM python:3.13.7-slim-bookworm
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
COPY contracts ./contracts
|
||||
COPY tests ./tests
|
||||
COPY student-tests ./student-tests
|
||||
COPY scripts ./scripts
|
||||
COPY lab.json requirements-dev.txt ./
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"]
|
||||
CMD ["python", "-m", "messenger.server"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence
|
||||
.PHONY: help check validate test load test-persistence test-own verify
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -24,3 +24,10 @@ load:
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-own:
|
||||
$(PYTHON) student-tests/test_api.py
|
||||
docker compose exec -T api python student-tests/test_database.py
|
||||
|
||||
verify:
|
||||
$(PYTHON) scripts/verify.py
|
||||
|
||||
@@ -1,37 +1,152 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: **Осокин Глеб**, группа **БВТ2302**, Gitea **glebossokin**.
|
||||
- Целевая оценка: **4**.
|
||||
- Основа: skeleton `65003b6`; ветка `solution/BVT2302-Osokin`.
|
||||
- HTTP-сервер и валидация адаптированы из собственной lab1; помощь ИИ использовалась для разбора требований, проверки результатов и подготовки отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Git, работающий Docker с Linux containers, Compose с поддержкой `--wait`, Python 3.9+ на клиентской машине. На Windows используется PowerShell, GNU Make необязателен. Runtime: Python **3.13.7** (`python:3.13.7-slim-bookworm`), PostgreSQL **17.6** (`postgres:17.6-bookworm`), Redis **7.4.5** (`redis:7.4.5-bookworm`). В [requirements.txt](requirements.txt) закреплены psycopg/psycopg-binary 3.2.10, psycopg-pool 3.2.6, redis-py 6.4.0, typing_extensions 4.15.0. Публичные проверки используют стандартную библиотеку Python.
|
||||
|
||||
Из корня `lab2`:
|
||||
|
||||
```powershell
|
||||
python scripts/bootstrap.py
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
Invoke-RestMethod http://127.0.0.1:8080/health/ready
|
||||
$env:BASE_URL = 'http://127.0.0.1:8080'
|
||||
python scripts/check_contract.py
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python scripts/verify.py
|
||||
```
|
||||
|
||||
`bootstrap.py` генерирует локальный случайный пароль PostgreSQL, URL подключения и постоянный ключ cursor в `secrets/`, создаёт `.env`. Повторный запуск сохраняет существующие значения. Эти файлы исключены из Git и Docker build context. Оператор должен сохранять `secrets/` между запусками с существующими volumes: новый пароль в файле не меняет пароль уже инициализированной PostgreSQL.
|
||||
|
||||
Compose дожидается PostgreSQL, запускает одноразовый `migrate`, после успешного завершения — API. Создаётся `schema_migrations` и атомарно применяется `migrations/001_initial.sql`. Повторный запуск сверяет SHA-256 применённых миграций и ничего не дублирует:
|
||||
|
||||
```powershell
|
||||
docker compose run --rm --no-deps migrate
|
||||
docker compose run --rm --no-deps migrate
|
||||
```
|
||||
|
||||
В Linux/macOS после запуска стенда доступны Make-эквиваленты:
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test BASE_URL=http://127.0.0.1:8080
|
||||
make test-own
|
||||
chmod +x scripts/restart.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
|
||||
make verify
|
||||
```
|
||||
|
||||
В этой Windows-среде `check`/`test` выполнялись соответствующими Python-командами; persistence использовал `restart.cmd`, вызывающий тот же `scripts/restart.py`, что и `restart.sh`. Публичные проверки не изменены. Полная валидация OpenAPI отдельно: `make validate` после установки `requirements-dev.txt`; это проверка контракта, а не самого API.
|
||||
|
||||
Безопасная остановка: `docker compose down`. Следующий `docker compose up -d --wait` использует прежние volumes. **Намеренное удаление всех учебных данных:** `docker compose down --volumes` — уничтожает пользователей, чаты, сообщения и сессии этого проекта. Для persistence так останавливать нельзя. Скрипты проверки не удаляют основные volumes.
|
||||
|
||||
Демонстрационный сценарий API: `python scripts/demo.py`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент → HTTP API → PostgreSQL и Redis. Проект таблиц, ограничений и транзакционных границ перед реализацией SQL: [infra/SCHEMA.md](infra/SCHEMA.md).
|
||||
|
||||
**PostgreSQL** — источник истины пользователей, password KDF, чатов, членства и сообщений. Есть UUID PK, UNIQUE username, FK, CHECK длины/формата. Составной FK сообщения на членство запрещает отправителя вне чата. Создание чата и всех участников — одна транзакция: FK violation отменяет весь результат. HTTP 201 регистрации/чата/сообщения выдаётся только после успешного COMMIT. Пользовательские значения передаются SQL-параметрами `%s`, не включаются в текст SQL.
|
||||
|
||||
**Redis** — единственное хранилище сессий, не кэш истории. Ключ `session:<sha256(token)>`, значение UUID пользователя. Токен имеет 256 бит случайности (`secrets.token_urlsafe(32)`), в исходном виде на сервере не сохраняется. `SET ... PXAT` назначает абсолютный expiry; чтение не продлевает TTL. Logout — DEL; expiry удаляет ключ автоматически. AOF включён с `appendfsync always`, `/data` находится на именованном volume `redisdata`. `maxmemory=64mb`, `noeviction`: при нехватке памяти вход завершится 503 вместо вытеснения действующих сессий. Redis не публикует порт на host; отсутствие Redis-аутентификации — учебное упрощение изолированной сети.
|
||||
|
||||
Пароль обрабатывается scrypt: N=16384, r=8, p=1, соль 16 bytes, dklen 32 bytes. Поддерживается 12–128 Unicode code points, включая двоеточие, без усечения по 72 bytes. Одинаковые пароли имеют разные salted hashes. Basic только для создания сессии, бизнес-endpoint требуют Bearer. `X-User-Id` игнорируется. Чужой и отсутствующий чат дают одинаковый 404 после аутентификации.
|
||||
|
||||
История не кэшируется: следующая операция чтения обращается к primary PostgreSQL и видит завершённый COMMIT. Индекс `messages_history(chat_id, created_at, id)` обслуживает keyset-пагинацию. Cursor содержит последнюю пару `(created_at,id)`, связан с чатом/пользователем и подписан HMAC постоянным ключом. Запрашивается `limit+1` строк; `next_cursor=null`, если продолжения нет. При конкурентных транзакциях поздний COMMIT с timestamp до уже пройденной позиции может быть пропущен обходом. Snapshot между HTTP-запросами и глобальный монотонный порядок COMMIT не обещаются; неизменная история обходится без потерь/повторов.
|
||||
|
||||
| Ресурс | Ограничение |
|
||||
| --- | --- |
|
||||
| HTTP workers / очередь | 16 / 32; переполнение → 503 |
|
||||
| HTTP socket / JSON body | 2 s / 65 536 bytes |
|
||||
| PostgreSQL pool | min 1, max 8, максимум 16 ожидающих, ожидание 1 s |
|
||||
| PostgreSQL connect / statement / lock | 2 s / 2 s / 1 s; idle transaction 5 s |
|
||||
| Redis pool | max 8, connect/read по 0.5 s, без автоматических retry |
|
||||
| DNS внутри API | timeout 1 s, attempts 1; учитывает остановленный Docker service |
|
||||
|
||||
`/health/live` проверяет процесс. `/health/ready` проверяет PostgreSQL и Redis: обе зависимости критичны для полного синхронного пути. При отказе возвращается JSON 503 с request_id/Retry-After, без SQL, credentials и stack trace. Анонимный пользователь не становится доверенным. Регистрация технически может работать без Redis, но полный API в этом состоянии неготов.
|
||||
|
||||
API работает от UID 10001. Логи содержат method/status/request_id/latency без текста сообщений и заголовков авторизации. SIGTERM прекращает приём, завершает ограниченные workers и закрывает пулы. PostgreSQL/Redis используют именованные volumes; API локального состояния не имеет. Из lab1 перенесён обязательный REST API, заменены in-memory структуры и X-User-Id; исследования уровня 5 не переносились.
|
||||
|
||||
### Соответствие .env.example
|
||||
|
||||
| Переменная | Реальное использование |
|
||||
| --- | --- |
|
||||
| APP_HOST | Bind внутри API, по умолчанию 0.0.0.0 |
|
||||
| APP_PORT | Публикуемый host port; внутри контейнера всегда 8080 |
|
||||
| APP_ENV | Маркер учебного окружения, не меняет хранение |
|
||||
| SESSION_TTL_SECONDS | TTL новых сессий, default 3600; для smoke/persistence ≥300 |
|
||||
| SESSION_STORE | В этом решении redis, Compose фиксирует выбор |
|
||||
| DATABASE_URL_FILE | `/run/secrets/database_url` из локального secrets/database_url |
|
||||
| CACHE_URL_FILE | `/run/secrets/cache_url` из secrets/cache_url |
|
||||
| CURSOR_KEY_FILE | `/run/secrets/cursor_key` из secrets/cursor_key |
|
||||
|
||||
Пути secrets в Compose фиксированы и совпадают с примером; значения URL не публикуются в tracked `.env.example`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Полный прогон: `python scripts/verify.py`. Все команды из корня `lab2`, при работающем стенде.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдение | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Структура контракта | `python scripts/check_contract.py` | PASS; публичные tests/contracts сохранены | [check.txt](evidence/check.txt) |
|
||||
| Полная валидация OpenAPI | `python scripts/validate_openapi.py` с requirements-dev | PASS: OpenAPI 3.1 и JSON Schema | [validate.txt](evidence/validate.txt) |
|
||||
| Запуск из чистой копии с нуля | `python scripts/clean_check.py` | Локальный git clone без .env/secrets, no-cache build, новые volumes, smoke и 3 проверки БД OK | [clean-clone.txt](evidence/clean-clone.txt) |
|
||||
| Smoke и минимум lab1 с новым auth | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS: lab2 public API scenario | [smoke.txt](evidence/smoke.txt) |
|
||||
| PostgreSQL, FK, UNIQUE, атомарность чата | `docker compose exec -T api python student-tests/test_database.py` | Ошибка второго участника: числа chats/chat_members не изменились | [database.txt](evidence/database.txt), [SQL](migrations/001_initial.sql) |
|
||||
| Миграции с нуля / повторно | `docker compose up -d --build --wait`; test_repeat_migrations | Первичная миграция; два повтора Already applied, количества сохранены | [database.txt](evidence/database.txt) |
|
||||
| Сохранность данных и сессий | Публичный resilience с restart.cmd / restart.sh | PASS, прежний токен читает прежнее сообщение; все 3 процесса перезапущены | [persistence.txt](evidence/persistence.txt), [restart.json](evidence/restart.json) |
|
||||
| Basic/Bearer/logout/X-User-Id | Smoke, собственные API-тесты | Неверный пароль/Basic на business/X-User-Id → 401; logout 204, затем 401 | [smoke.txt](evidence/smoke.txt), [own-api.txt](evidence/own-api.txt) |
|
||||
| Password KDF | test_password_salted_kdf; test_unicode_password_and_logout | Разные scrypt hashes одинаковых паролей; пароль 128 Unicode code points работает | [database.txt](evidence/database.txt), [own-api.txt](evidence/own-api.txt) |
|
||||
| Единый запуск / volumes / secrets вне Git | `docker compose --env-file .env config --quiet` | Конфигурация успешна; 3 сервиса и одноразовая миграция | [compose.txt](evidence/compose.txt), [compose.yaml](compose.yaml) |
|
||||
| Redis с TTL и persistence | `python scripts/check_expiry.py`; persistence | TTL=2: сначала 200, через 2.2 s → 401; обычная сессия пережила restart | [expiry.txt](evidence/expiry.txt), [redis.conf](infra/redis.conf) |
|
||||
| Настройки durability | SHOW fsync/synchronous_commit/full_page_writes; Redis CONFIG GET | PostgreSQL: on/on/on; Redis: appendonly=yes, appendfsync=always | [PostgreSQL](evidence/durability-postgres.txt), [Redis](evidence/durability-redis.txt) |
|
||||
| SQL parameters / bounded pools / timeouts | Реальный HTTP-путь, отказ зависимостей | 503 в пределах бюджета вместо зависания | [outages.txt](evidence/outages.txt), [core.py](src/messenger/core.py) |
|
||||
| Гонка username | test_registration_race, 10 клиентов | Ровно 1×201 и 9×409 | [own-api.txt](evidence/own-api.txt) |
|
||||
| EXPLAIN ANALYZE BUFFERS на ≥10 000 | `docker compose exec -T api python scripts/explain_history.py` | Index Scan messages_history, 51 строка, 4 shared hit blocks, без Seq Scan | [explain.json](evidence/explain.json) |
|
||||
| Автоматические отказы Redis/БД | `python student-tests/test_outages.py` | По 3×503; live 200, ready 503; старые сессия/сообщение сохранены после восстановления | [outages.txt](evidence/outages.txt) |
|
||||
| Собственные негативные/конкурентные тесты | `python student-tests/test_api.py` | 4 теста OK; 100 уникальных сообщений от 10 клиентов и обход страниц | [own-api.txt](evidence/own-api.txt) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Проверялась корректность уровня 4: durable stores сохраняют подтверждённое сообщение/сессию при перезапуске, а потеря зависимости ограниченно блокирует защищённые операции. Это не исследовательский трек 5 и не измерение максимальной нагрузки.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Среда: Windows 10 build 19045, Docker Desktop Engine 29.8.2, Linux/WSL2 x86_64, 8 логических CPU и около 7.71 GiB RAM для Docker. Лимиты: API 2 CPU/512 MiB, PostgreSQL 2 CPU/512 MiB, Redis 1 CPU/128 MiB. Клиент Python 3.14.3, runtime 3.13.7. Digest хранилищ и сведения о машине: [environment.json](evidence/environment.json).
|
||||
|
||||
Профиль: последовательный smoke; гонка регистрации 10 клиентов; 100 POST сообщений от 10 клиентов; expiry; по три последовательных чтения при отказе зависимости; один план на новом чате с 10 000 коротких сообщений, cursor после позиции 9000, limit=50. Прогрев — создание fixtures и ANALYZE; план измерен на уже прогретых страницах. Повторяемость обеспечена скриптами. Статистические серии throughput не проводились, поэтому RPS/p50/p95/p99 не заявляются. [resources.txt](evidence/resources.txt) — единичный снимок CPU/RAM после проверок, не среднее/пик под нагрузкой.
|
||||
|
||||
| Воздействие | Ожидаем | Наблюдали | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Restart api/db/redis | Старый токен и сообщение действуют после ready | Persistence PASS; StartedAt всех 3 изменились, volumes прежние | В пределах лимитов публичного сценария; потерь нет |
|
||||
| TTL=2 в отдельном API | После expiry → 401 | До expiry 200, через 2.2 s — 401 | Основной TTL=3600 не менялся |
|
||||
| Logout | Токен недействителен | 204, затем 401 | Новый вход создаёт новую сессию |
|
||||
| Redis stop | 503; анонимный вход запрещён | 3 ошибки за 1.005–1.027 s; X-User-Id без сессии → 401 | 0.604 s от начала start до ready/me; сессия/сообщение сохранены |
|
||||
| PostgreSQL stop | 503; процесс жив | 3 ошибки за 1.005–1.008 s; live 200, ready 503 | 4.599 s от начала start до ready/me; сессия/сообщение сохранены |
|
||||
| Неизвестный участник после INSERT chats | Полный rollback | Числа chats/chat_members до/после равны | Частичного чата нет |
|
||||
|
||||
SQL Execution Time **0.049 ms**, Index Scan отдал 51 строку, Shared Hit Blocks=4, Shared Read Blocks=0. Это один локальный прогретый SQL-запрос без HTTP/аутентификации: подтверждение использования индекса, не SLA. Не сравнивались альтернативные индексы/100 000 сообщений/backup — это задания оценки 5.
|
||||
|
||||
Первый сценарий отказа выявил DNS-ожидание остановленного контейнера дольше socket timeout. В итоговый Compose добавлены `dns_opt: timeout:1, attempts:1`; результаты выше получены после исправления.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один host, по одному PostgreSQL/Redis: проверяемый restart выдерживается, уничтожение диска и HA не покрыты. При отказе хранилища бизнес-путь временно недоступен.
|
||||
- FLUSHDB/удаление Redis volume/повреждение AOF теряют сессии; нужно войти заново. PostgreSQL volume содержит бизнес-данные. Потеря cursor_key аннулирует прежние cursors.
|
||||
- После COMMIT может оборваться транспорт: клиент не получит 201, хотя сообщение записано. Идемпотентность появляется в lab3 и здесь не реализована.
|
||||
- Уже аутентифицированный конкурентный запрос может завершиться после logout; последующие проверки токена получают 401.
|
||||
- HTTP опубликован только на 127.0.0.1, данные синтетические. Для удалённого использования нужны TLS и усиление конфигурации. В учебной БД используется роль, созданная официальным образом; production role management не реализован.
|
||||
- Snapshot между страницами и упорядочивание конкурентных COMMIT не обещаются.
|
||||
- Не реализованы треки 5: исследование согласованности кэша/stampede с измерениями, сравнение индексов на ≥100 000 и backup/restore.
|
||||
|
||||
## Источники
|
||||
|
||||
- PostgreSQL: [транзакции](https://www.postgresql.org/docs/17/tutorial-transactions.html), [WAL durability](https://www.postgresql.org/docs/17/wal-reliability.html), [EXPLAIN](https://www.postgresql.org/docs/17/using-explain.html).
|
||||
- Redis: [persistence и AOF](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [PEXPIREAT](https://redis.io/docs/latest/commands/pexpireat/).
|
||||
- Psycopg: [пулы](https://www.psycopg.org/psycopg3/docs/advanced/pool.html), [SQL parameters](https://www.psycopg.org/psycopg3/docs/basic/params.html).
|
||||
- Python: [hashlib.scrypt](https://docs.python.org/3.13/library/hashlib.html#hashlib.scrypt), [secrets](https://docs.python.org/3.13/library/secrets.html).
|
||||
- [RFC 7617 — Basic](https://www.rfc-editor.org/rfc/rfc7617.html), контракт и публичные проверки этого репозитория, собственная lab1, помощь OpenAI Codex, раскрытая в начале отчёта.
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
name: mtusi-lab2-osokin
|
||||
x-app: &app
|
||||
image: mtusi-messenger:lab2
|
||||
build: .
|
||||
environment: &app-env
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: redis
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
CURSOR_KEY_FILE: /run/secrets/cursor_key
|
||||
secrets: [database_url, cache_url, cursor_key]
|
||||
mem_limit: 512m
|
||||
cpus: 2
|
||||
dns_opt: ['timeout:1', 'attempts:1']
|
||||
services:
|
||||
db:
|
||||
image: postgres:17.6-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
secrets: [db_password]
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 2
|
||||
redis:
|
||||
image: redis:7.4.5-bookworm
|
||||
command: [redis-server, /usr/local/etc/redis/redis.conf]
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 1
|
||||
migrate:
|
||||
<<: *app
|
||||
command: [python, -m, messenger.migrate]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
api:
|
||||
<<: *app
|
||||
ports: ['127.0.0.1:${APP_PORT:-8080}:8080']
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
stop_grace_period: 8s
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
secrets:
|
||||
db_password: {file: ./secrets/db_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
cursor_key: {file: ./secrets/cursor_key}
|
||||
@@ -0,0 +1,2 @@
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,100 @@
|
||||
$ git clone --no-hardlinks <workspace> <workspace>\output\clean-clone
|
||||
Cloning into 'C:\Users\blobb\OneDrive\Документи\ChatGPT\высоконагружноут\lab2\output\clean-clone'...
|
||||
done.
|
||||
|
||||
$ python scripts/bootstrap.py
|
||||
Local .env and secrets ready (existing values preserved).
|
||||
|
||||
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env build --no-cache --quiet
|
||||
Image mtusi-messenger:lab2 Building
|
||||
Image mtusi-messenger:lab2 Building
|
||||
Image mtusi-messenger:lab2 Built
|
||||
Image mtusi-messenger:lab2 Built
|
||||
|
||||
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env up -d --wait
|
||||
Volume mtusi-lab2-osokin-clean_pgdata Creating
|
||||
Network mtusi-lab2-osokin-clean_default Creating
|
||||
Volume mtusi-lab2-osokin-clean_redisdata Creating
|
||||
Volume mtusi-lab2-osokin-clean_pgdata Creating
|
||||
Volume mtusi-lab2-osokin-clean_redisdata Creating
|
||||
Network mtusi-lab2-osokin-clean_default Creating
|
||||
Volume mtusi-lab2-osokin-clean_pgdata Created
|
||||
Volume mtusi-lab2-osokin-clean_pgdata Created
|
||||
Volume mtusi-lab2-osokin-clean_redisdata Created
|
||||
Volume mtusi-lab2-osokin-clean_redisdata Created
|
||||
Network mtusi-lab2-osokin-clean_default Created
|
||||
Network mtusi-lab2-osokin-clean_default Created
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Creating
|
||||
Container mtusi-lab2-osokin-clean-db-1 Creating
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Created
|
||||
Container mtusi-lab2-osokin-clean-db-1 Created
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Creating
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Created
|
||||
Container mtusi-lab2-osokin-clean-api-1 Creating
|
||||
Container mtusi-lab2-osokin-clean-api-1 Created
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Starting
|
||||
Container mtusi-lab2-osokin-clean-db-1 Starting
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Started
|
||||
Container mtusi-lab2-osokin-clean-db-1 Started
|
||||
Container mtusi-lab2-osokin-clean-db-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-db-1 Healthy
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Starting
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Started
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Healthy
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Exited
|
||||
Container mtusi-lab2-osokin-clean-api-1 Starting
|
||||
Container mtusi-lab2-osokin-clean-api-1 Started
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-api-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-db-1 Waiting
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Healthy
|
||||
Container mtusi-lab2-osokin-clean-db-1 Healthy
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Exited
|
||||
Container mtusi-lab2-osokin-clean-api-1 Healthy
|
||||
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:18082
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env exec -T api python student-tests/test_database.py
|
||||
test_password_salted_kdf (__main__.Database.test_password_salted_kdf) ... ok
|
||||
test_repeat_migrations (__main__.Database.test_repeat_migrations) ... Already applied: 001_initial.sql
|
||||
Migrations committed.
|
||||
Already applied: 001_initial.sql
|
||||
Migrations committed.
|
||||
ok
|
||||
test_rollback_after_chat_insert (__main__.Database.test_rollback_after_chat_insert) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 3 tests in 2.993s
|
||||
|
||||
OK
|
||||
|
||||
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env logs --no-log-prefix migrate
|
||||
Applied: 001_initial.sql
|
||||
Migrations committed.
|
||||
|
||||
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env down
|
||||
Container mtusi-lab2-osokin-clean-api-1 Stopping
|
||||
Container mtusi-lab2-osokin-clean-api-1 Stopped
|
||||
Container mtusi-lab2-osokin-clean-api-1 Removing
|
||||
Container mtusi-lab2-osokin-clean-api-1 Removed
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Stopping
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Stopping
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Stopped
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Removing
|
||||
Container mtusi-lab2-osokin-clean-migrate-1 Removed
|
||||
Container mtusi-lab2-osokin-clean-db-1 Stopping
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Stopped
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Removing
|
||||
Container mtusi-lab2-osokin-clean-redis-1 Removed
|
||||
Container mtusi-lab2-osokin-clean-db-1 Stopped
|
||||
Container mtusi-lab2-osokin-clean-db-1 Removing
|
||||
Container mtusi-lab2-osokin-clean-db-1 Removed
|
||||
Network mtusi-lab2-osokin-clean_default Removing
|
||||
Network mtusi-lab2-osokin-clean_default Removed
|
||||
@@ -0,0 +1 @@
|
||||
$ docker compose --env-file .env config --quiet
|
||||
@@ -0,0 +1,13 @@
|
||||
$ docker compose exec -T api python student-tests/test_database.py
|
||||
test_password_salted_kdf (__main__.Database.test_password_salted_kdf) ... ok
|
||||
test_repeat_migrations (__main__.Database.test_repeat_migrations) ... Already applied: 001_initial.sql
|
||||
Migrations committed.
|
||||
Already applied: 001_initial.sql
|
||||
Migrations committed.
|
||||
ok
|
||||
test_rollback_after_chat_insert (__main__.Database.test_rollback_after_chat_insert) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 3 tests in 2.310s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,8 @@
|
||||
$ python scripts/demo.py
|
||||
1. Readiness 200; registered three users; Basic login returned Bearer sessions.
|
||||
2. Created chat: c2817969-606b-4551-bff4-c98ba084c1b4 ; members: Alice and Bob.
|
||||
3. POST message -> 201; message id: ac5d70bf-4bd4-4da8-a17b-e8505600bc15
|
||||
4. Bob reads Alice's committed message -> 200: Привет, Боб! Это лабораторная 2.
|
||||
5. Eve is authenticated but not a chat member -> 404.
|
||||
6. X-User-Id without Bearer -> 401.
|
||||
7. Logout -> 204; previous token -> 401. Demo PASS.
|
||||
@@ -0,0 +1,15 @@
|
||||
$ docker compose exec -T db psql -U messenger -d messenger -c SHOW fsync; -c SHOW synchronous_commit; -c SHOW full_page_writes;
|
||||
fsync
|
||||
-------
|
||||
on
|
||||
(1 row)
|
||||
|
||||
synchronous_commit
|
||||
--------------------
|
||||
on
|
||||
(1 row)
|
||||
|
||||
full_page_writes
|
||||
------------------
|
||||
on
|
||||
(1 row)
|
||||
@@ -0,0 +1,5 @@
|
||||
$ docker compose exec -T redis redis-cli CONFIG GET appendonly appendfsync
|
||||
appendfsync
|
||||
always
|
||||
appendonly
|
||||
yes
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"verified_at_utc": "2026-10-26T12:25:05.273386+00:00",
|
||||
"os": "Windows-10-10.0.19045-SP0",
|
||||
"client_python": "3.14.3",
|
||||
"docker": {
|
||||
"ServerVersion": "29.8.2",
|
||||
"OperatingSystem": "Docker Desktop",
|
||||
"KernelVersion": "6.18.40.1-microsoft-standard-WSL2",
|
||||
"NCPU": 8,
|
||||
"MemTotal": 8275755008,
|
||||
"Architecture": "x86_64"
|
||||
},
|
||||
"images": {
|
||||
"postgres:17.6-bookworm": [
|
||||
"postgres@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3"
|
||||
],
|
||||
"redis:7.4.5-bookworm": [
|
||||
"redis@sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb"
|
||||
]
|
||||
},
|
||||
"runtime_python": "Python 3.13.7"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
$ python scripts/check_expiry.py
|
||||
Container mtusi-lab2-osokin-api-run-caf7bb185347 Creating
|
||||
Container mtusi-lab2-osokin-api-run-caf7bb185347 Created
|
||||
PASS: session accepted before expiry; rejected after 2.2s with SESSION_TTL_SECONDS=2.
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"messages": 10000,
|
||||
"cursor_position": 9000,
|
||||
"page_limit": 50,
|
||||
"fetched_with_lookahead": 51,
|
||||
"sql": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s",
|
||||
"explain": [
|
||||
{
|
||||
"Plan": {
|
||||
"Node Type": "Limit",
|
||||
"Parallel Aware": false,
|
||||
"Async Capable": false,
|
||||
"Startup Cost": 0.29,
|
||||
"Total Cost": 17.77,
|
||||
"Plan Rows": 51,
|
||||
"Plan Width": 68,
|
||||
"Actual Startup Time": 0.008,
|
||||
"Actual Total Time": 0.025,
|
||||
"Actual Rows": 51,
|
||||
"Actual Loops": 1,
|
||||
"Shared Hit Blocks": 4,
|
||||
"Shared Read Blocks": 0,
|
||||
"Shared Dirtied Blocks": 0,
|
||||
"Shared Written Blocks": 0,
|
||||
"Local Hit Blocks": 0,
|
||||
"Local Read Blocks": 0,
|
||||
"Local Dirtied Blocks": 0,
|
||||
"Local Written Blocks": 0,
|
||||
"Temp Read Blocks": 0,
|
||||
"Temp Written Blocks": 0,
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "Index Scan",
|
||||
"Parent Relationship": "Outer",
|
||||
"Parallel Aware": false,
|
||||
"Async Capable": false,
|
||||
"Scan Direction": "Forward",
|
||||
"Index Name": "messages_history",
|
||||
"Relation Name": "messages",
|
||||
"Alias": "messages",
|
||||
"Startup Cost": 0.29,
|
||||
"Total Cost": 473.36,
|
||||
"Plan Rows": 1380,
|
||||
"Plan Width": 68,
|
||||
"Actual Startup Time": 0.007,
|
||||
"Actual Total Time": 0.021,
|
||||
"Actual Rows": 51,
|
||||
"Actual Loops": 1,
|
||||
"Index Cond": "((chat_id = '137c9dcf-9f70-465c-862c-595c9a50d5c3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.009+00'::timestamp with time zone, '3873da86-1a3e-2129-c84e-1daf5c084a0d'::uuid)))",
|
||||
"Rows Removed by Index Recheck": 0,
|
||||
"Shared Hit Blocks": 4,
|
||||
"Shared Read Blocks": 0,
|
||||
"Shared Dirtied Blocks": 0,
|
||||
"Shared Written Blocks": 0,
|
||||
"Local Hit Blocks": 0,
|
||||
"Local Read Blocks": 0,
|
||||
"Local Dirtied Blocks": 0,
|
||||
"Local Written Blocks": 0,
|
||||
"Temp Read Blocks": 0,
|
||||
"Temp Written Blocks": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"Planning": {
|
||||
"Shared Hit Blocks": 6,
|
||||
"Shared Read Blocks": 0,
|
||||
"Shared Dirtied Blocks": 0,
|
||||
"Shared Written Blocks": 0,
|
||||
"Local Hit Blocks": 0,
|
||||
"Local Read Blocks": 0,
|
||||
"Local Dirtied Blocks": 0,
|
||||
"Local Written Blocks": 0,
|
||||
"Temp Read Blocks": 0,
|
||||
"Temp Written Blocks": 0
|
||||
},
|
||||
"Planning Time": 0.079,
|
||||
"Triggers": [],
|
||||
"Execution Time": 0.049
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
$ python student-tests/test_outages.py
|
||||
Container mtusi-lab2-osokin-redis-1 Stopping
|
||||
Container mtusi-lab2-osokin-redis-1 Stopped
|
||||
Container mtusi-lab2-osokin-redis-1 Starting
|
||||
Container mtusi-lab2-osokin-redis-1 Started
|
||||
Container mtusi-lab2-osokin-db-1 Stopping
|
||||
Container mtusi-lab2-osokin-db-1 Stopped
|
||||
Container mtusi-lab2-osokin-db-1 Starting
|
||||
Container mtusi-lab2-osokin-db-1 Started
|
||||
[
|
||||
{
|
||||
"service": "redis",
|
||||
"errors": 3,
|
||||
"status": 503,
|
||||
"seconds": [
|
||||
1.0267414001282305,
|
||||
1.0045876000076532,
|
||||
1.007952299900353
|
||||
],
|
||||
"recovery_seconds": 0.604,
|
||||
"original_session_preserved": true,
|
||||
"original_message_preserved": true,
|
||||
"anonymous_status": 401
|
||||
},
|
||||
{
|
||||
"service": "db",
|
||||
"errors": 3,
|
||||
"status": 503,
|
||||
"seconds": [
|
||||
1.0076619999017566,
|
||||
1.0045177999418229,
|
||||
1.0052553000859916
|
||||
],
|
||||
"recovery_seconds": 4.599,
|
||||
"original_session_preserved": true,
|
||||
"original_message_preserved": true,
|
||||
"anonymous_status": 401
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
$ python student-tests/test_api.py
|
||||
test_concurrent_messages (__main__.API.test_concurrent_messages) ... ok
|
||||
test_registration_race (__main__.API.test_registration_race) ... ok
|
||||
test_unicode_password_and_logout (__main__.API.test_unicode_password_and_logout) ... ok
|
||||
test_validation_access_and_pagination (__main__.API.test_validation_access_and_pagination) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 tests in 4.016s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,3 @@
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
$ docker stats --no-stream --format {{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}} mtusi-lab2-osokin-api-1 mtusi-lab2-osokin-db-1 mtusi-lab2-osokin-redis-1
|
||||
mtusi-lab2-osokin-api-1 CPU=51.26% RAM=158.7MiB / 512MiB
|
||||
mtusi-lab2-osokin-db-1 CPU=8.94% RAM=27.76MiB / 512MiB
|
||||
mtusi-lab2-osokin-redis-1 CPU=0.44% RAM=4.301MiB / 128MiB
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"before": {
|
||||
"api": {
|
||||
"id": "b6b4e6f8c09ee890165395f2af5e40945dd74ce270a9c35c09b946db8ed160ef",
|
||||
"started_at": "2026-10-26T20:13:56.437921569Z",
|
||||
"mounts": []
|
||||
},
|
||||
"db": {
|
||||
"id": "9a25c8ce8dd3a0a64d4def2f27765f792fb00739774c0e0926fe0cb1017c27fd",
|
||||
"started_at": "2026-10-26T20:13:49.316227457Z",
|
||||
"mounts": [
|
||||
{
|
||||
"name": "lab2-defense-0eaa9a8d_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
"redis": {
|
||||
"id": "8009d787a3e25c88d024c121ff5dd00a613e77b785a30fbe4b08b1dcb57a021e",
|
||||
"started_at": "2026-10-26T20:13:49.903677339Z",
|
||||
"mounts": [
|
||||
{
|
||||
"name": "lab2-defense-0eaa9a8d_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"api": {
|
||||
"id": "b6b4e6f8c09ee890165395f2af5e40945dd74ce270a9c35c09b946db8ed160ef",
|
||||
"started_at": "2026-10-26T20:15:24.833818504Z",
|
||||
"mounts": []
|
||||
},
|
||||
"db": {
|
||||
"id": "9a25c8ce8dd3a0a64d4def2f27765f792fb00739774c0e0926fe0cb1017c27fd",
|
||||
"started_at": "2026-10-26T20:15:24.683887137Z",
|
||||
"mounts": [
|
||||
{
|
||||
"name": "lab2-defense-0eaa9a8d_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
"redis": {
|
||||
"id": "8009d787a3e25c88d024c121ff5dd00a613e77b785a30fbe4b08b1dcb57a021e",
|
||||
"started_at": "2026-10-26T20:15:24.575559122Z",
|
||||
"mounts": [
|
||||
{
|
||||
"name": "lab2-defense-0eaa9a8d_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,2 @@
|
||||
$ ../lab1/.venv/Scripts/python.exe scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Схема и транзакции (проект перед реализацией SQL)
|
||||
|
||||
Источник истины бизнес-данных — PostgreSQL; единственное хранилище сессий — Redis.
|
||||
|
||||
* users: UUID PK, username UNIQUE с CHECK формата, display_name, password_hash (scrypt), created_at timestamptz.
|
||||
* chats: UUID PK, title, created_at timestamptz.
|
||||
* chat_members: (chat_id, user_id) PK, FK к chats и users; position для сохранения порядка участников.
|
||||
* messages: UUID PK, FK chat_id → chats; составной FK (chat_id, sender_id) → chat_members.
|
||||
* schema_migrations: имя миграции PK, SHA-256 содержимого, applied_at. DDL и запись версии в одной транзакции под advisory lock.
|
||||
|
||||
Индексы: UNIQUE username защищает конкурентную регистрацию. messages_history(chat_id, created_at, id) обслуживает `WHERE chat_id = $1 AND (created_at,id) > ($2,$3) ORDER BY created_at,id LIMIT $4`, без OFFSET. chat_members_user(user_id,chat_id) — список доступных чатов. chats_order(created_at,id) — порядок чатов.
|
||||
|
||||
Создание чата: INSERT chats + INSERT всех участников в одной транзакции; неизвестный участник вызывает FK violation и rollback всего чата. Регистрация и отправка сообщения — отдельные транзакции; HTTP 201 только после успешного COMMIT. Чтения — READ COMMITTED с primary PostgreSQL; история не кэшируется.
|
||||
|
||||
Redis: `session:<sha256(token)>` → UUID пользователя, абсолютный expiry (PXAT); TTL не обновляется при чтении. Logout — DEL. AOF + appendfsync always, именованный volume. Потеря Redis: 503, никакого доверия X-User-Id; restart сохраняет действующие сессии, FLUSHDB отзывает все. Истечение/DEL не требует инвалидации других кэшей.
|
||||
|
||||
Cursor подписан HMAC-SHA256 постоянным локальным ключом, связан с chat_id либо user_id; хранит (created_at,id). Это позиция, не snapshot: конкурентная поздно закоммиченная запись со старым timestamp может оказаться за уже пройденным cursor. Неизменная история обходится без повторов; UUID разрешает равные timestamps. Часы назначает PostgreSQL, монотонность между транзакциями не обещается.
|
||||
@@ -0,0 +1,10 @@
|
||||
bind 0.0.0.0
|
||||
protected-mode no
|
||||
port 6379
|
||||
dir /data
|
||||
appendonly yes
|
||||
appendfsync always
|
||||
save ""
|
||||
maxmemory 64mb
|
||||
maxmemory-policy noeviction
|
||||
# Only the private Compose network can reach Redis; no host port is published.
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX chat_members_user ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_order ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages(chat_id, created_at, id);
|
||||
@@ -0,0 +1,5 @@
|
||||
psycopg==3.2.10
|
||||
psycopg-binary==3.2.10
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.4.0
|
||||
typing_extensions==4.15.0
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once; never print them or replace existing files."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root / 'secrets'
|
||||
folder.mkdir(exist_ok=True)
|
||||
password_file = folder / 'db_password'
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(24), encoding='utf-8')
|
||||
password = password_file.read_text(encoding='utf-8').strip()
|
||||
values = {
|
||||
'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
'cursor_key': secrets.token_hex(32),
|
||||
}
|
||||
for name, value in values.items():
|
||||
target = folder / name
|
||||
if not target.exists():
|
||||
target.write_text(value, encoding='utf-8')
|
||||
if not (root / '.env').exists():
|
||||
(root / '.env').write_text('APP_HOST=0.0.0.0\nAPP_PORT=8080\nAPP_ENV=lab\nSESSION_TTL_SECONDS=3600\nSESSION_STORE=redis\n', encoding='utf-8')
|
||||
print('Local .env and secrets ready (existing values preserved).')
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Separate API container with 2 second TTL; main API configuration is unchanged."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
command = ['docker', 'compose', 'run', '-d', '--no-deps', '-e', 'SESSION_TTL_SECONDS=2', 'api']
|
||||
identifier = subprocess.check_output(command, cwd=root, text=True).strip()
|
||||
try:
|
||||
# Start the client in the temporary container after its HTTP server is ready.
|
||||
subprocess.run(['docker', 'exec', identifier, 'python', '-c',
|
||||
"import time,urllib.request; "
|
||||
"\nfor _ in range(50):\n try:\n urllib.request.urlopen('http://localhost:8080/health/ready',timeout=3); break"
|
||||
"\n except OSError: time.sleep(.2)\nelse: raise RuntimeError('API not ready')"], check=True)
|
||||
subprocess.run(['docker', 'exec', identifier, 'python', 'student-tests/test_expiry.py'], check=True)
|
||||
finally:
|
||||
subprocess.run(['docker', 'rm', '-f', identifier], check=True, stdout=subprocess.DEVNULL)
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check committed sources in a new local clone with fresh secrets and new volumes."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
clone = ROOT / 'output' / 'clean-clone'
|
||||
project = 'mtusi-lab2-osokin-clean'
|
||||
env = dict(os.environ, PYTHONIOENCODING='utf-8', BASE_URL='http://127.0.0.1:18082')
|
||||
|
||||
|
||||
def main():
|
||||
if clone.exists():
|
||||
raise SystemExit('output/clean-clone already exists; preserve or move it before repeating this check.')
|
||||
clone.parent.mkdir(exist_ok=True)
|
||||
log = []
|
||||
def run(command, cwd):
|
||||
log.append('$ ' + ' '.join(command).replace(str(ROOT), '<workspace>').replace(sys.executable, 'python'))
|
||||
process = subprocess.run(command, cwd=cwd, env=env, text=True, encoding='utf-8', errors='replace',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
log.append(process.stdout)
|
||||
cleaned = '\n'.join(line.rstrip() for line in '\n'.join(log).splitlines()) + '\n'
|
||||
(ROOT / 'evidence' / 'clean-clone.txt').write_text(cleaned, encoding='utf-8')
|
||||
if process.returncode:
|
||||
raise SystemExit('Clean clone check failed; see evidence/clean-clone.txt')
|
||||
run(['git', 'clone', '--no-hardlinks', str(ROOT), str(clone)], ROOT)
|
||||
assert not (clone / '.env').exists()
|
||||
assert not (clone / 'secrets').exists()
|
||||
run([sys.executable, 'scripts/bootstrap.py'], clone)
|
||||
local_env = clone / '.env'
|
||||
local_env.write_text(local_env.read_text(encoding='utf-8').replace('APP_PORT=8080', 'APP_PORT=18082'), encoding='utf-8')
|
||||
compose = ['docker', 'compose', '-p', project, '--env-file', '.env']
|
||||
try:
|
||||
run(compose + ['build', '--no-cache', '--quiet'], clone)
|
||||
run(compose + ['up', '-d', '--wait'], clone)
|
||||
run([sys.executable, 'scripts/check_contract.py'], clone)
|
||||
run([sys.executable, 'tests/smoke.py', '--base-url', env['BASE_URL']], clone)
|
||||
run(compose + ['exec', '-T', 'api', 'python', 'student-tests/test_database.py'], clone)
|
||||
run(compose + ['logs', '--no-log-prefix', 'migrate'], clone)
|
||||
finally:
|
||||
# Keep even the disposable check's volumes; only stop its isolated containers.
|
||||
run(compose + ['down'], clone)
|
||||
print('PASS: clean local clone; no copied secrets, fresh volumes, no-cache build, smoke/database checks.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Human-readable demonstration. Passwords and session tokens stay in memory."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
print('1. Readiness 200; registered three users; Basic login returned Bearer sessions.')
|
||||
print('2. Created chat:', chat['id'], '; members: Alice and Bob.')
|
||||
message = c.message(alice, chat, 'Привет, Боб! Это лабораторная 2.')['json']
|
||||
print('3. POST message -> 201; message id:', message['id'])
|
||||
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json']
|
||||
assert page['items'] == [message]
|
||||
print('4. Bob reads Alice\'s committed message -> 200:', message['text'])
|
||||
c.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve)
|
||||
print('5. Eve is authenticated but not a chat member -> 404.')
|
||||
c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': alice['id']})
|
||||
print('6. X-User-Id without Bearer -> 401.')
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
||||
print('7. Logout -> 204; previous token -> 401. Demo PASS.')
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One index, 10,000 messages; grade 4 evidence, not a grade 5 comparison."""
|
||||
import json
|
||||
import uuid
|
||||
from messenger.core import Config, HISTORY_SQL, Store
|
||||
|
||||
store = Store(Config.from_env())
|
||||
try:
|
||||
actor = store.create_user(dict(username='plan_' + uuid.uuid4().hex[:20],
|
||||
display_name='Plan fixture', password='synthetic-plan-password'))['id']
|
||||
chat = store.create_chat(actor, dict(title='EXPLAIN 10000', member_ids=[actor]))
|
||||
with store.pool.connection() as conn:
|
||||
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT md5(%s || i::text)::uuid, %s, %s, 'fixture ' || i, "
|
||||
"timestamptz '2026-01-01 UTC' + i * interval '1 microsecond' "
|
||||
"FROM generate_series(1,10000) i", (chat['id'], chat['id'], actor))
|
||||
conn.execute('ANALYZE messages')
|
||||
key = conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s '
|
||||
'ORDER BY created_at,id LIMIT 1 OFFSET 8999', (chat['id'],)).fetchone()
|
||||
plan = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + HISTORY_SQL,
|
||||
(chat['id'], key['created_at'], key['id'], 51)).fetchone()['QUERY PLAN']
|
||||
count = conn.execute('SELECT count(*) AS n FROM messages WHERE chat_id=%s', (chat['id'],)).fetchone()['n']
|
||||
# OFFSET is used once to choose a fixture position, never in the API query.
|
||||
assert count == 10000
|
||||
nodes = []
|
||||
def walk(node):
|
||||
nodes.append(node)
|
||||
for child in node.get('Plans', []):
|
||||
walk(child)
|
||||
walk(plan[0]['Plan'])
|
||||
assert any(n.get('Index Name') == 'messages_history' for n in nodes)
|
||||
assert all(n['Node Type'] != 'Seq Scan' for n in nodes)
|
||||
print(json.dumps(dict(messages=count, cursor_position=9000, page_limit=50, fetched_with_lookahead=51,
|
||||
sql=HISTORY_SQL, explain=plan), ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\restart.py
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restart precisely this lab's API, PostgreSQL and Redis, keeping both volumes."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, text=True)
|
||||
|
||||
|
||||
def states():
|
||||
result = {}
|
||||
for service in ('api', 'db', 'redis'):
|
||||
identifier = compose('ps', '-q', service).strip()
|
||||
data = json.loads(subprocess.check_output(['docker', 'inspect', identifier], text=True))[0]
|
||||
result[service] = dict(id=identifier, started_at=data['State']['StartedAt'],
|
||||
mounts=[dict(name=m.get('Name'), destination=m['Destination'])
|
||||
for m in data['Mounts'] if m['Type'] == 'volume'])
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
before = states()
|
||||
subprocess.run(['docker', 'compose', '--env-file', '.env', 'restart', 'api', 'db', 'redis'],
|
||||
cwd=ROOT, check=True)
|
||||
after = states()
|
||||
for service in before:
|
||||
assert before[service]['started_at'] != after[service]['started_at'], service + ' did not restart'
|
||||
assert before[service]['mounts'] == after[service]['mounts'], service + ' changed volumes'
|
||||
(ROOT / 'evidence' / 'restart.json').write_text(json.dumps(dict(before=before, after=after), indent=2) + '\n')
|
||||
print('Restarted api/db/redis; volume names unchanged. Evidence: evidence/restart.json')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/restart.py
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the grade 4 checks and preserve small sanitized evidence files."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / 'evidence'
|
||||
env = dict(os.environ, BASE_URL=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'), PYTHONIOENCODING='utf-8')
|
||||
python = sys.executable
|
||||
if hasattr(sys.stdout, 'reconfigure'):
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
|
||||
def run(name, command):
|
||||
print('Running', name, flush=True)
|
||||
process = subprocess.run(command, cwd=ROOT, env=env, text=True, encoding='utf-8', errors='replace',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
log = '$ ' + ' '.join(command).replace(python, 'python') + '\n' + process.stdout
|
||||
log = '\n'.join(line.rstrip() for line in log.splitlines()).rstrip() + '\n'
|
||||
(EVIDENCE / (name + '.txt')).write_text(log, encoding='utf-8')
|
||||
print(process.stdout, end='', flush=True)
|
||||
if process.returncode:
|
||||
raise SystemExit('Check failed: ' + name)
|
||||
|
||||
|
||||
def main():
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
run('check', [python, 'scripts/check_contract.py'])
|
||||
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
|
||||
run('smoke', [python, 'tests/smoke.py', '--base-url', env['BASE_URL']])
|
||||
run('own-api', [python, 'student-tests/test_api.py'])
|
||||
run('database', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_database.py'])
|
||||
run('persistence', [python, 'tests/resilience.py', '--mode', 'persistence', '--action-script',
|
||||
'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh', '--base-url', env['BASE_URL']])
|
||||
run('expiry', [python, 'scripts/check_expiry.py'])
|
||||
run('outages', [python, 'student-tests/test_outages.py'])
|
||||
run('demo', [python, 'scripts/demo.py'])
|
||||
run('durability-postgres', ['docker', 'compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger',
|
||||
'-d', 'messenger', '-c', 'SHOW fsync;', '-c', 'SHOW synchronous_commit;',
|
||||
'-c', 'SHOW full_page_writes;'])
|
||||
run('durability-redis', ['docker', 'compose', 'exec', '-T', 'redis', 'redis-cli',
|
||||
'CONFIG', 'GET', 'appendonly', 'appendfsync'])
|
||||
output = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python',
|
||||
'scripts/explain_history.py'], cwd=ROOT, text=True)
|
||||
plan = json.loads(output)
|
||||
(EVIDENCE / 'explain.json').write_text(json.dumps(plan, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: EXPLAIN on', plan['messages'], 'messages uses messages_history.', flush=True)
|
||||
run('resources', ['docker', 'stats', '--no-stream', '--format',
|
||||
'{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}',
|
||||
'mtusi-lab2-osokin-api-1', 'mtusi-lab2-osokin-db-1', 'mtusi-lab2-osokin-redis-1'])
|
||||
environment = dict(verified_at_utc=datetime.now(timezone.utc).isoformat(), os=platform.platform(),
|
||||
client_python=platform.python_version())
|
||||
environment['docker'] = json.loads(subprocess.check_output(
|
||||
['docker', 'info', '--format', '{{json .}}'], text=True))
|
||||
# Whitelist machine properties; never persist Docker client config/proxy details.
|
||||
environment['docker'] = {k: environment['docker'][k] for k in
|
||||
('ServerVersion', 'OperatingSystem', 'KernelVersion', 'NCPU', 'MemTotal', 'Architecture')}
|
||||
environment['images'] = {}
|
||||
environment['runtime_python'] = subprocess.check_output(
|
||||
['docker', 'compose', 'exec', '-T', 'api', 'python', '--version'], cwd=ROOT, text=True).strip()
|
||||
for image in ('postgres:17.6-bookworm', 'redis:7.4.5-bookworm'):
|
||||
environment['images'][image] = json.loads(subprocess.check_output(
|
||||
['docker', 'image', 'inspect', image, '--format', '{{json .RepoDigests}}'], text=True))
|
||||
(EVIDENCE / 'environment.json').write_text(json.dumps(environment, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: all grade 4 scenarios completed; no grade 5 track was run.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Lab 1: bounded, single-process in-memory messenger."""
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Validation and persistent state. HTTP 201 is emitted after transaction exit/COMMIT."""
|
||||
import base64
|
||||
from dataclasses import dataclass, fields
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
import redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, message):
|
||||
self.status, self.message = status, message
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
app_host: str = '0.0.0.0'
|
||||
app_port: int = 8080
|
||||
workers: int = 16
|
||||
request_queue: int = 32
|
||||
socket_timeout: int = 2
|
||||
max_body_bytes: int = 65536
|
||||
session_ttl_seconds: int = 3600
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
values = {}
|
||||
for field in fields(cls):
|
||||
raw = os.environ.get(field.name.upper(), field.default)
|
||||
values[field.name] = str(raw) if field.type is str else int(raw)
|
||||
if field.type is int and values[field.name] < 1:
|
||||
raise ValueError(field.name + ' must be positive')
|
||||
result = cls(**values)
|
||||
if result.app_port > 65535 or result.workers > 32 or result.request_queue > 128:
|
||||
raise ValueError('port/workers/queue outside bounds')
|
||||
if result.max_body_bytes > 1048576 or result.socket_timeout > 5:
|
||||
raise ValueError('body <= 1 MiB and socket timeout <= 5 required')
|
||||
if os.getenv('SESSION_STORE', 'redis') != 'redis':
|
||||
raise ValueError('This solution uses Redis sessions')
|
||||
return result
|
||||
|
||||
|
||||
def secret_file(name):
|
||||
return Path(os.environ[name]).read_text(encoding='utf-8').strip()
|
||||
|
||||
|
||||
def db_connect_kwargs():
|
||||
return dict(connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1000 '
|
||||
'-c idle_in_transaction_session_timeout=5000', row_factory=dict_row)
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value):
|
||||
raise APIError(400, 'Invalid UUID')
|
||||
return str(uuid.UUID(value))
|
||||
|
||||
|
||||
def string(value, maximum):
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum or '\x00' in value:
|
||||
raise APIError(400, 'Invalid string')
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise APIError(400, 'Invalid Unicode') from None
|
||||
return value
|
||||
|
||||
|
||||
def object_fields(body, expected):
|
||||
if not isinstance(body, dict) or set(body) != set(expected):
|
||||
raise APIError(400, 'Invalid object fields')
|
||||
|
||||
|
||||
def password_value(value):
|
||||
if not isinstance(value, str) or not value.strip() or not 12 <= len(value) <= 128:
|
||||
raise APIError(400, 'Invalid password length')
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise APIError(400, 'Invalid Unicode') from None
|
||||
return value
|
||||
|
||||
|
||||
def password_hash(value, salt=None):
|
||||
salt = salt or secrets.token_bytes(16)
|
||||
digest = hashlib.scrypt(value.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32)
|
||||
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
|
||||
|
||||
|
||||
def password_matches(value, encoded):
|
||||
# Only this versioned format is generated by registration.
|
||||
salt = bytes.fromhex(encoded.split('$')[4])
|
||||
return hmac.compare_digest(password_hash(value, salt), encoded)
|
||||
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def public_row(row):
|
||||
return {key: (str(value) if isinstance(value, uuid.UUID) else stamp(value)
|
||||
if isinstance(value, datetime) else value)
|
||||
for key, value in row.items() if key != 'password_hash'}
|
||||
|
||||
|
||||
USER_COLUMNS = 'id, username, display_name, created_at'
|
||||
MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at'
|
||||
# Shared by the real history endpoint and EXPLAIN evidence; all values are parameters.
|
||||
HISTORY_SQL = ('SELECT ' + MESSAGE_COLUMNS + ' FROM messages WHERE chat_id = %s '
|
||||
'AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s')
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.secret = bytes.fromhex(secret_file('CURSOR_KEY_FILE'))
|
||||
if len(self.secret) < 32:
|
||||
raise ValueError('Cursor key must be at least 256 bits')
|
||||
self.pool = ConnectionPool(secret_file('DATABASE_URL_FILE'), min_size=1, max_size=8,
|
||||
timeout=1, max_waiting=16, kwargs=db_connect_kwargs(),
|
||||
check=ConnectionPool.check_connection, open=True)
|
||||
self.cache_pool = redis.ConnectionPool.from_url(
|
||||
secret_file('CACHE_URL_FILE'), max_connections=8, socket_connect_timeout=.5,
|
||||
socket_timeout=.5, decode_responses=True, retry=Retry(NoBackoff(), 0))
|
||||
self.cache = redis.Redis(connection_pool=self.cache_pool)
|
||||
self.dummy_hash = password_hash('dummy-password-for-timing')
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=2)
|
||||
self.cache_pool.disconnect()
|
||||
|
||||
def ready(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
self.cache.ping()
|
||||
|
||||
def create_user(self, body):
|
||||
object_fields(body, ('username', 'display_name', 'password'))
|
||||
username = string(body['username'], 32)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
|
||||
raise APIError(400, 'Invalid username')
|
||||
display = string(body['display_name'], 100)
|
||||
hashed = password_hash(password_value(body['password']))
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO users(id, username, display_name, password_hash) '
|
||||
'VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS,
|
||||
(uuid.uuid4(), username, display, hashed)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, 'Username already exists') from None
|
||||
return public_row(row)
|
||||
|
||||
def login(self, authorization):
|
||||
try:
|
||||
scheme, encoded = authorization.split(' ', 1)
|
||||
if scheme.lower() != 'basic' or len(encoded) > 1024:
|
||||
raise ValueError()
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
|
||||
raise ValueError()
|
||||
password_value(password)
|
||||
except (AttributeError, ValueError, UnicodeError, APIError):
|
||||
raise APIError(401, 'Invalid credentials') from None
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT * FROM users WHERE username = %s', (username,)).fetchone()
|
||||
matched = password_matches(password, row['password_hash'] if row else self.dummy_hash)
|
||||
if not matched or row is None:
|
||||
raise APIError(401, 'Invalid credentials')
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.config.session_ttl_seconds * 1000
|
||||
self.cache.set(self.session_key(token), str(row['id']), pxat=expires_ms)
|
||||
return dict(token=token, token_type='Bearer',
|
||||
expires_at=stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)),
|
||||
user=public_row(row))
|
||||
|
||||
@staticmethod
|
||||
def session_key(token):
|
||||
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
|
||||
|
||||
def auth(self, header):
|
||||
try:
|
||||
scheme, token = header.split(' ', 1)
|
||||
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise ValueError()
|
||||
except (ValueError, AttributeError):
|
||||
raise APIError(401, 'Invalid session') from None
|
||||
key = self.session_key(token)
|
||||
actor = self.cache.get(key)
|
||||
if actor is None:
|
||||
raise APIError(401, 'Invalid session')
|
||||
return actor, key
|
||||
|
||||
def me(self, actor):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id = %s', (actor,)).fetchone()
|
||||
if row is None:
|
||||
raise APIError(401, 'Invalid session')
|
||||
return public_row(row)
|
||||
|
||||
def chat_access(self, actor, identifier):
|
||||
identifier = uuid_value(identifier)
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s',
|
||||
(identifier, actor)).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, 'Object not found')
|
||||
return identifier
|
||||
|
||||
@staticmethod
|
||||
def chat_row(conn, row):
|
||||
members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position',
|
||||
(row['id'],)).fetchall()
|
||||
return dict(public_row(row), member_ids=[str(x['user_id']) for x in members])
|
||||
|
||||
def chat(self, chat_id):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()
|
||||
return self.chat_row(conn, row)
|
||||
|
||||
def create_chat(self, actor, body):
|
||||
object_fields(body, ('title', 'member_ids'))
|
||||
title = string(body['title'], 100)
|
||||
members = body['member_ids']
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise APIError(400, 'Invalid members')
|
||||
members = [uuid_value(x) for x in members]
|
||||
if len(members) != len(set(members)) or actor not in members:
|
||||
raise APIError(400, 'Invalid members')
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *',
|
||||
(uuid.uuid4(), title)).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)',
|
||||
[(row['id'], member, pos) for pos, member in enumerate(members)])
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise APIError(400, 'Invalid members') from None
|
||||
return dict(public_row(row), member_ids=members)
|
||||
|
||||
def create_message(self, actor, chat_id, body):
|
||||
object_fields(body, ('text',))
|
||||
text = string(body['text'], 4000)
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
|
||||
'RETURNING ' + MESSAGE_COLUMNS, (uuid.uuid4(), chat_id, actor, text)).fetchone()
|
||||
return public_row(row)
|
||||
|
||||
def encode_cursor(self, scope, key):
|
||||
payload = json.dumps([1, scope, *key], separators=(',', ':')).encode()
|
||||
return base64.urlsafe_b64encode(payload + hmac.digest(self.secret, payload, 'sha256')).rstrip(b'=').decode()
|
||||
|
||||
def decode_cursor(self, cursor, scope):
|
||||
try:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', cursor):
|
||||
raise ValueError()
|
||||
raw = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_', validate=True)
|
||||
if base64.urlsafe_b64encode(raw).rstrip(b'=').decode() != cursor:
|
||||
raise ValueError()
|
||||
payload, signature = raw[:-32], raw[-32:]
|
||||
if not hmac.compare_digest(signature, hmac.digest(self.secret, payload, 'sha256')):
|
||||
raise ValueError()
|
||||
version, resource, timestamp, identifier = json.loads(payload)
|
||||
if version != 1 or resource != scope or not isinstance(timestamp, str) or not timestamp.endswith('Z'):
|
||||
raise ValueError()
|
||||
return datetime.fromisoformat(timestamp.replace('Z', '+00:00')), uuid_value(identifier)
|
||||
except (ValueError, TypeError, UnicodeError, APIError):
|
||||
raise APIError(400, 'Invalid cursor') from None
|
||||
|
||||
def page(self, actor, chat_id, query):
|
||||
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
|
||||
raise APIError(400, 'Invalid query')
|
||||
limit = query.get('limit', ['50'])[0]
|
||||
if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100:
|
||||
raise APIError(400, 'Invalid limit')
|
||||
limit = int(limit)
|
||||
scope = 'messages:' + chat_id if chat_id else 'chats:' + actor
|
||||
key = self.decode_cursor(query['cursor'][0], scope) if 'cursor' in query else (
|
||||
datetime.min.replace(tzinfo=timezone.utc), '00000000-0000-0000-0000-000000000000')
|
||||
with self.pool.connection() as conn:
|
||||
if chat_id:
|
||||
rows = conn.execute(HISTORY_SQL, (chat_id, *key, limit + 1)).fetchall()
|
||||
items = [public_row(row) for row in rows[:limit]]
|
||||
else:
|
||||
rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id '
|
||||
'WHERE m.user_id=%s AND (c.created_at,c.id) > (%s,%s) '
|
||||
'ORDER BY c.created_at,c.id LIMIT %s', (actor, *key, limit + 1)).fetchall()
|
||||
items = [self.chat_row(conn, row) for row in rows[:limit]]
|
||||
cursor = self.encode_cursor(scope, (items[-1]['created_at'], items[-1]['id'])) if len(rows) > limit else None
|
||||
return dict(items=items, next_cursor=cursor)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Transactional, versioned migrations; repeat application is a no-op."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from .core import secret_file, db_connect_kwargs
|
||||
|
||||
|
||||
def main():
|
||||
with psycopg.connect(secret_file('DATABASE_URL_FILE'), **db_connect_kwargs()) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2026, 2)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(name text PRIMARY KEY, sha256 text NOT NULL, '
|
||||
'applied_at timestamptz NOT NULL DEFAULT clock_timestamp())')
|
||||
for path in sorted((Path(__file__).resolve().parents[2] / 'migrations').glob('*.sql')):
|
||||
content = path.read_bytes()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
row = conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s', (path.name,)).fetchone()
|
||||
if row:
|
||||
if row['sha256'] != digest:
|
||||
raise RuntimeError('Applied migration checksum mismatch')
|
||||
print('Already applied:', path.name)
|
||||
continue
|
||||
conn.execute(content.decode('utf-8'), prepare=False)
|
||||
conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)', (path.name, digest))
|
||||
print('Applied:', path.name)
|
||||
print('Migrations committed.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Bounded HTTP worker pool; one request per connection; JSON logs omit bodies."""
|
||||
from collections import Counter
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
import redis
|
||||
|
||||
from .core import APIError, Config, Store
|
||||
|
||||
|
||||
CODES = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict",
|
||||
413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited",
|
||||
503: "unavailable", 500: "internal_error"}
|
||||
|
||||
|
||||
def error_body(status, message, request_id):
|
||||
return {"error": {"code": CODES.get(status, "invalid_request"),
|
||||
"message": message, "request_id": request_id}}
|
||||
|
||||
|
||||
def log_event(**data):
|
||||
logging.info(json.dumps(data, ensure_ascii=True))
|
||||
|
||||
|
||||
class Server(HTTPServer):
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, config):
|
||||
self.config, self.store = config, Store(config)
|
||||
self.jobs = queue.Queue(maxsize=config.request_queue)
|
||||
self.draining = threading.Event()
|
||||
self.stats_lock = threading.Lock()
|
||||
self.statuses = Counter()
|
||||
self.active = 0
|
||||
self.clients = set()
|
||||
self.started = time.monotonic()
|
||||
self.request_queue_size = config.request_queue
|
||||
super().__init__((config.app_host, config.app_port), Handler)
|
||||
self.threads = [threading.Thread(target=self.worker, name="http-" + str(n), daemon=True)
|
||||
for n in range(config.workers)]
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def get_request(self):
|
||||
client, address = super().get_request()
|
||||
client.settimeout(self.config.socket_timeout)
|
||||
return client, address
|
||||
|
||||
def record(self, status):
|
||||
with self.stats_lock:
|
||||
self.statuses[str(status)] += 1
|
||||
|
||||
def process_request(self, request, address):
|
||||
try:
|
||||
if self.draining.is_set():
|
||||
raise queue.Full()
|
||||
self.jobs.put_nowait((request, address))
|
||||
except queue.Full:
|
||||
# Transport admission before HTTP parsing: bounded time, no extra thread.
|
||||
identifier = str(uuid.uuid4())
|
||||
body = json.dumps(error_body(503, "Request queue full or shutting down", identifier)).encode()
|
||||
headers = ("HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
|
||||
"Connection: close\r\nRetry-After: 1\r\nX-Request-Id: " + identifier +
|
||||
"\r\nContent-Length: " + str(len(body)) + "\r\n\r\n").encode()
|
||||
try:
|
||||
request.settimeout(.05)
|
||||
# Wait for the first request bytes: closing before a local client
|
||||
# finishes sendall can otherwise discard the 503 with a TCP reset.
|
||||
try:
|
||||
request.recv(16384)
|
||||
except OSError:
|
||||
pass
|
||||
request.sendall(headers + body)
|
||||
request.shutdown(socket.SHUT_WR)
|
||||
request.settimeout(.01)
|
||||
for _ in range(5):
|
||||
if not request.recv(16384):
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
self.shutdown_request(request)
|
||||
self.record(503)
|
||||
log_event(event="queue_rejected", status=503, request_id=identifier)
|
||||
|
||||
def worker(self):
|
||||
while not self.draining.is_set():
|
||||
try:
|
||||
request, address = self.jobs.get(timeout=.1)
|
||||
except queue.Empty:
|
||||
continue
|
||||
with self.stats_lock:
|
||||
self.active += 1
|
||||
self.clients.add(request)
|
||||
try:
|
||||
self.finish_request(request, address)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception:
|
||||
log_event(event="worker_error")
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
with self.stats_lock:
|
||||
self.active -= 1
|
||||
self.clients.discard(request)
|
||||
self.jobs.task_done()
|
||||
|
||||
def server_close(self):
|
||||
self.draining.set()
|
||||
with self.stats_lock:
|
||||
clients = list(self.clients)
|
||||
for client in clients:
|
||||
try:
|
||||
client.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
while True:
|
||||
try:
|
||||
request, _ = self.jobs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self.shutdown_request(request)
|
||||
self.jobs.task_done()
|
||||
deadline = time.monotonic() + 5
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=max(0, deadline - time.monotonic()))
|
||||
self.store.close()
|
||||
super().server_close()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "Messenger"
|
||||
sys_version = ""
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.request_id = str(uuid.uuid4())
|
||||
self.received = time.monotonic()
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def parse_request(self):
|
||||
original = self.rfile
|
||||
|
||||
class HeaderBudget:
|
||||
remaining = 16384
|
||||
|
||||
def readline(self, limit=-1):
|
||||
line = original.readline(min(limit, self.remaining + 1) if limit >= 0 else self.remaining + 1)
|
||||
self.remaining -= len(line)
|
||||
if self.remaining < 0:
|
||||
raise http.client.LineTooLong("header section")
|
||||
return line
|
||||
|
||||
self.rfile = HeaderBudget()
|
||||
try:
|
||||
return super().parse_request()
|
||||
finally:
|
||||
self.rfile = original
|
||||
|
||||
def send_error(self, code, message=None, explain=None):
|
||||
self.reply(code if code in CODES else 400, error_body(
|
||||
code if code in CODES else 400, "Invalid HTTP request", self.request_id))
|
||||
|
||||
def reply(self, status, body):
|
||||
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
|
||||
content_type = 'application/json'
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.send_header("X-Request-Id", self.request_id)
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Connection", "close")
|
||||
if status == 401:
|
||||
challenge = ('Basic realm="messenger", charset="UTF-8"'
|
||||
if urlsplit(self.path).path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
|
||||
self.send_header('WWW-Authenticate', challenge)
|
||||
if status in (429, 503):
|
||||
self.send_header("Retry-After", "1")
|
||||
self.end_headers()
|
||||
self.close_connection = True
|
||||
self.wfile.write(raw)
|
||||
self.server.record(status)
|
||||
# No URL query, header, username, text, or exception detail in logs.
|
||||
log_event(event="request", method=self.command, status=status,
|
||||
request_id=self.request_id, elapsed_ms=round(1000 * (time.monotonic() - self.received), 3))
|
||||
|
||||
def handle_expect_100(self):
|
||||
# Do not acknowledge a large unauthorized request before checking identity.
|
||||
return True
|
||||
|
||||
def body(self):
|
||||
if self.headers.get_all("Transfer-Encoding"):
|
||||
raise APIError(400, "Transfer-Encoding is unsupported")
|
||||
sizes = self.headers.get_all("Content-Length", [])
|
||||
if len(sizes) != 1 or not sizes[0].isascii() or not sizes[0].isdigit() or len(sizes[0]) > 10:
|
||||
raise APIError(400, "One valid Content-Length is required")
|
||||
size = int(sizes[0])
|
||||
if size > self.server.config.max_body_bytes:
|
||||
raise APIError(413, "Request body exceeds byte limit")
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise APIError(415, "Expected application/json")
|
||||
if self.headers.get("Expect", "").lower() == "100-continue":
|
||||
self.send_response_only(100)
|
||||
self.end_headers()
|
||||
raw = self.rfile.read(size)
|
||||
if len(raw) != size:
|
||||
raise APIError(400, "Incomplete request body")
|
||||
|
||||
def pairs(values):
|
||||
result = {}
|
||||
for key, value in values:
|
||||
if key in result:
|
||||
raise ValueError("duplicate key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
def constant(_):
|
||||
raise ValueError("non JSON constant")
|
||||
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=constant)
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
raise APIError(400, "Invalid JSON") from None
|
||||
|
||||
def dispatch(self):
|
||||
try:
|
||||
url = urlsplit(self.path)
|
||||
except ValueError:
|
||||
raise APIError(400, "Invalid request target") from None
|
||||
path, method = url.path, self.command
|
||||
store = self.server.store
|
||||
if self.server.draining.is_set():
|
||||
raise APIError(503, "Shutting down")
|
||||
if method == 'GET' and path == '/health/live':
|
||||
return 200, {'status': 'ok'}
|
||||
if method == 'GET' and path == '/health/ready':
|
||||
store.ready()
|
||||
return 200, {'status': 'ok'}
|
||||
if path == '/api/v1/users' and method == 'POST':
|
||||
return 201, store.create_user(self.body())
|
||||
headers = self.headers.get_all('Authorization', [])
|
||||
authorization = headers[0] if len(headers) == 1 else None
|
||||
if path == '/api/v1/auth/sessions' and method == 'POST':
|
||||
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
||||
raise APIError(400, 'Login takes no request body')
|
||||
return 201, store.login(authorization)
|
||||
actor, session_key = store.auth(authorization)
|
||||
if method == 'DELETE' and path == '/api/v1/auth/sessions/current':
|
||||
store.cache.delete(session_key)
|
||||
return 204, None
|
||||
if method == 'GET' and path == '/api/v1/users/me':
|
||||
return 200, store.me(actor)
|
||||
parts = path.strip("/").split("/")
|
||||
chat_id = None
|
||||
if len(parts) >= 4 and parts[:3] == ["api", "v1", "chats"]:
|
||||
chat_id = store.chat_access(actor, parts[3])
|
||||
if method == "POST" and path == "/api/v1/chats":
|
||||
return 201, store.create_chat(actor, self.body())
|
||||
if method == "POST" and chat_id and len(parts) == 5 and parts[4] == "messages":
|
||||
return 201, store.create_message(actor, chat_id, self.body())
|
||||
if method == "GET":
|
||||
if chat_id and len(parts) == 4:
|
||||
return 200, store.chat(chat_id)
|
||||
if path == "/api/v1/chats" or (chat_id and len(parts) == 5 and parts[4] == "messages"):
|
||||
try:
|
||||
query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True, max_num_fields=8)
|
||||
except ValueError:
|
||||
raise APIError(400, "Invalid query") from None
|
||||
return 200, store.page(actor, chat_id, query)
|
||||
raise APIError(404, "Object not found")
|
||||
|
||||
def handle_api(self):
|
||||
try:
|
||||
status, result = self.dispatch()
|
||||
except APIError as error:
|
||||
status = error.status
|
||||
result = error_body(status, error.message, self.request_id)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable, PoolTimeout, TooManyRequests, redis.RedisError):
|
||||
status, result = 503, error_body(503, 'Dependency temporarily unavailable', self.request_id)
|
||||
except (TimeoutError, socket.timeout):
|
||||
status, result = 400, error_body(400, "Request read timeout", self.request_id)
|
||||
except Exception:
|
||||
status, result = 500, error_body(500, "Internal error", self.request_id)
|
||||
log_event(event="internal_error", request_id=self.request_id)
|
||||
try:
|
||||
self.reply(status, result)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = do_OPTIONS = handle_api
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
# Pool diagnostics may include connection details; application emits sanitized errors.
|
||||
logging.getLogger('psycopg.pool').disabled = True
|
||||
config = Config.from_env()
|
||||
server = Server(config)
|
||||
|
||||
def stop(signum, _):
|
||||
if not server.draining.is_set():
|
||||
server.draining.set()
|
||||
log_event(event="shutdown_started", signal=signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
log_event(event="started", port=config.app_port, workers=config.workers)
|
||||
try:
|
||||
server.serve_forever(poll_interval=.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
log_event(event="shutdown_complete")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Own black-box tests; synthetic data, standard library, no tokens in output."""
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
class API(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.c = Client()
|
||||
|
||||
def test_registration_race(self):
|
||||
body = dict(username='race_' + uuid.uuid4().hex[:20], display_name='Гонка', password='Unicode:пароль🔑123')
|
||||
def register(_):
|
||||
return self.c.call('POST', '/api/v1/users', [201, 409], body=body)['status']
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = list(pool.map(register, range(10)))
|
||||
self.assertEqual(statuses.count(201), 1)
|
||||
self.assertEqual(statuses.count(409), 9)
|
||||
|
||||
def test_validation_access_and_pagination(self):
|
||||
self.c.call('POST', '/api/v1/users', 400,
|
||||
body=dict(username='spaces_' + uuid.uuid4().hex[:20], display_name='Invalid', password=' ' * 12))
|
||||
alice, bob, eve, chat = self.c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id']
|
||||
for raw in (b'{', b'{"text":NaN}', b'{"text":"a","text":"b"}', b'[]'):
|
||||
self.c.call('POST', path + '/messages', 400, actor=alice, raw=raw,
|
||||
headers={'Content-Type': 'application/json'})
|
||||
for body in ({'text': ' '}, {'text': 'hi', 'sender_id': eve['id']}, {'text': 2}, {'text': '\x00'}):
|
||||
self.c.call('POST', path + '/messages', 400, actor=alice, body=body)
|
||||
self.c.call('POST', path + '/messages', 404, actor=eve, body={'text': 'нет доступа'})
|
||||
self.c.call('POST', '/api/v1/users', 409, body=alice['registration'])
|
||||
for members in ([alice['id'], str(uuid.uuid4())], [bob['id']], [alice['id'], alice['id']]):
|
||||
self.c.call('POST', '/api/v1/chats', 400, actor=alice,
|
||||
body={'title': 'Invalid membership', 'member_ids': members})
|
||||
sent = [self.c.message(alice, chat, 'Страница ' + str(i))['json'] for i in range(7)]
|
||||
read, cursor = [], None
|
||||
while True:
|
||||
query = '?limit=2' + ('&cursor=' + quote(cursor, safe='') if cursor else '')
|
||||
page = self.c.call('GET', path + '/messages' + query, actor=bob, schema='MessagePage')['json']
|
||||
read.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
break
|
||||
self.assertEqual(read, sorted(sent, key=lambda x: (x['created_at'], x['id'])))
|
||||
first = self.c.call('GET', path + '/messages?limit=1', actor=alice, schema='MessagePage')['json']
|
||||
other = self.c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
self.c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + first['next_cursor'], 400, actor=alice)
|
||||
self.c.call('GET', path + '/messages?cursor=broken', 400, actor=alice)
|
||||
|
||||
def test_concurrent_messages(self):
|
||||
alice, _, _, chat = self.c.bootstrap()
|
||||
def send(i):
|
||||
return self.c.message(alice, chat, 'Concurrent ' + str(i))['json']['id']
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
ids = list(pool.map(send, range(100)))
|
||||
page = self.c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
||||
actor=alice, schema='MessagePage')['json']
|
||||
self.assertEqual(len(set(ids)), 100)
|
||||
self.assertEqual({x['id'] for x in page['items']}, set(ids))
|
||||
self.assertIsNone(page['next_cursor'])
|
||||
|
||||
def test_unicode_password_and_logout(self):
|
||||
username = 'utf_' + uuid.uuid4().hex[:20]
|
||||
password = '🔑:пароль' * 16 # 128 code points, far more than bcrypt's 72 bytes
|
||||
self.assertEqual(len(password), 128)
|
||||
self.c.call('POST', '/api/v1/users', 201,
|
||||
body=dict(username=username, display_name='Unicode', password=password), schema='User')
|
||||
def login(value, expected):
|
||||
auth = base64.b64encode((username + ':' + value).encode()).decode()
|
||||
return self.c.call('POST', '/api/v1/auth/sessions', expected,
|
||||
headers={'Authorization': 'Basic ' + auth}, schema='Session' if expected == 201 else None)
|
||||
actor = {'token': login(password, 201)['json']['token']}
|
||||
login('incorrect-password', 401)
|
||||
self.c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
self.c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
self.c.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Run inside API container: directly observe rollback and stored password KDF."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import subprocess
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
|
||||
from messenger.core import APIError, Config, Store, password_matches
|
||||
|
||||
|
||||
class Database(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.store = Store(Config.from_env())
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.store.close()
|
||||
|
||||
def test_rollback_after_chat_insert(self):
|
||||
actor = self.store.create_user(dict(username='rollback_' + uuid.uuid4().hex[:16],
|
||||
display_name='Rollback', password='rollback-password'))['id']
|
||||
with self.store.pool.connection() as conn:
|
||||
before = conn.execute('SELECT count(*) AS n FROM chats').fetchone()['n']
|
||||
members_before = conn.execute('SELECT count(*) AS n FROM chat_members').fetchone()['n']
|
||||
# First membership succeeds, second violates FK after the chat was inserted.
|
||||
with self.assertRaises(APIError) as raised:
|
||||
self.store.create_chat(actor, dict(title='Must rollback', member_ids=[actor, str(uuid.uuid4())]))
|
||||
self.assertEqual(raised.exception.status, 400)
|
||||
with self.store.pool.connection() as conn:
|
||||
self.assertEqual(conn.execute('SELECT count(*) AS n FROM chats').fetchone()['n'], before)
|
||||
self.assertEqual(conn.execute('SELECT count(*) AS n FROM chat_members').fetchone()['n'], members_before)
|
||||
|
||||
def test_password_salted_kdf(self):
|
||||
password = 'identical-password'
|
||||
ids = [self.store.create_user(dict(username='salt_' + uuid.uuid4().hex[:20],
|
||||
display_name='Salt', password=password))['id'] for _ in range(2)]
|
||||
with self.store.pool.connection() as conn:
|
||||
hashes = [conn.execute('SELECT password_hash FROM users WHERE id=%s', (i,)).fetchone()['password_hash'] for i in ids]
|
||||
self.assertNotEqual(*hashes)
|
||||
for value in hashes:
|
||||
self.assertTrue(value.startswith('scrypt$16384$8$1$'))
|
||||
self.assertTrue(password_matches(password, value))
|
||||
self.assertNotEqual(value, password)
|
||||
|
||||
def test_repeat_migrations(self):
|
||||
def counts():
|
||||
with self.store.pool.connection() as conn:
|
||||
return [conn.execute('SELECT count(*) AS n FROM ' + table).fetchone()['n']
|
||||
for table in ('users', 'chats', 'chat_members', 'messages', 'schema_migrations')]
|
||||
before = counts()
|
||||
for _ in range(2):
|
||||
subprocess.run([sys.executable, '-m', 'messenger.migrate'], check=True)
|
||||
self.assertEqual(counts(), before)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Use a separate API process with SESSION_TTL_SECONDS=2, sharing real stores."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check, timestamp
|
||||
|
||||
c = Client()
|
||||
actor = c.user('expiry')
|
||||
response = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
time.sleep(2.2)
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
print('PASS: session accepted before expiry; rejected after 2.2s with SESSION_TTL_SECONDS=2.')
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Host-side operator test: stop one lab dependency, assert bounded failures, restore."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
|
||||
c = Client(timeout=4)
|
||||
actor, _, _, chat = c.bootstrap()
|
||||
original = c.message(actor, chat, 'Before dependency outage')['json']
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
summary = []
|
||||
for service in ('redis', 'db'):
|
||||
subprocess.run(['docker', 'compose', 'stop', service], cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
timings = []
|
||||
for _ in range(3):
|
||||
response = c.call('GET', '/api/v1/users/me', 503, actor=actor)
|
||||
check(response['json']['error']['code'] == 'unavailable', 'wrong dependency error')
|
||||
check(response['elapsed'] < 3.5, 'failure exceeded response budget')
|
||||
timings.append(response['elapsed'])
|
||||
c.call('POST', path, 503, actor=actor, body={'text': 'Must not commit while unavailable'})
|
||||
c.call('GET', '/health/ready', 503)
|
||||
c.call('GET', '/health/live', schema='Health')
|
||||
c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
|
||||
c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Bearer ' + 'A' * 43}) if service == 'db' else c.call(
|
||||
'GET', '/api/v1/users/me', 503, headers={'Authorization': 'Bearer ' + 'A' * 43})
|
||||
finally:
|
||||
started = time.monotonic()
|
||||
subprocess.run(['docker', 'compose', 'start', service], cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'dependency did not recover in 30 seconds')
|
||||
time.sleep(.25)
|
||||
recovery = time.monotonic() - started
|
||||
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
check(page['items'] == [original], 'lost confirmation or wrote during dependency outage')
|
||||
summary.append(dict(service=service, errors=3, status=503, seconds=timings,
|
||||
recovery_seconds=round(recovery, 3), original_session_preserved=True,
|
||||
original_message_preserved=True, anonymous_status=401))
|
||||
print(json.dumps(summary, indent=2))
|
||||
Reference in New Issue
Block a user