47 changed files with 1839 additions and 19 deletions
+1
View File
@@ -17,4 +17,5 @@ backups
__pycache__
node_modules
evidence
output
.DS_Store
+2 -1
View File
@@ -7,5 +7,6 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
CURSOR_KEY_FILE=/run/secrets/cursor_key
+5
View File
@@ -0,0 +1,5 @@
* text=auto
*.sh text eol=lf
*.py text eol=lf
*.sql text eol=lf
*.conf text eol=lf
+28
View File
@@ -0,0 +1,28 @@
name: Lab 2 application
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Local synthetic secrets and stack
run: |
python scripts/bootstrap.py
docker compose --env-file .env up -d --build --wait
- name: Grade 4 checks
run: make verify PYTHON=python
- name: Evidence without secrets
if: always()
uses: actions/upload-artifact@v4
with:
name: lab2-evidence
path: evidence/
- name: Stop stack, preserve volumes
if: always()
run: docker compose down
+5
View File
@@ -28,5 +28,10 @@ coverage/
.DS_Store
.idea/
.vscode/
output/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
# Personal study notes stay local
DEFENSE.md
DEFENSE_STEPS.md
MR_DRAFT.md
+16
View File
@@ -0,0 +1,16 @@
FROM python:3.13.7-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app
COPY src ./src
COPY migrations ./migrations
COPY contracts ./contracts
COPY tests ./tests
COPY student-tests ./student-tests
COPY scripts ./scripts
COPY lab.json requirements-dev.txt ./
USER app
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"]
CMD ["python", "-m", "messenger.server"]
+8 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence
.PHONY: help check validate test load test-persistence test-own verify
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -24,3 +24,10 @@ load:
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-own:
$(PYTHON) student-tests/test_api.py
docker compose exec -T api python student-tests/test_database.py
verify:
$(PYTHON) scripts/verify.py
+132 -17
View File
@@ -1,37 +1,152 @@
# Отчёт — лабораторная 2
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: **Осокин Глеб**, группа **БВТ2302**, Gitea **glebossokin**.
- Целевая оценка: **4**.
- Основа: skeleton `65003b6`; ветка `solution/BVT2302-Osokin`.
- HTTP-сервер и валидация адаптированы из собственной lab1; помощь ИИ использовалась для разбора требований, проверки результатов и подготовки отчёта.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Git, работающий Docker с Linux containers, Compose с поддержкой `--wait`, Python 3.9+ на клиентской машине. На Windows используется PowerShell, GNU Make необязателен. Runtime: Python **3.13.7** (`python:3.13.7-slim-bookworm`), PostgreSQL **17.6** (`postgres:17.6-bookworm`), Redis **7.4.5** (`redis:7.4.5-bookworm`). В [requirements.txt](requirements.txt) закреплены psycopg/psycopg-binary 3.2.10, psycopg-pool 3.2.6, redis-py 6.4.0, typing_extensions 4.15.0. Публичные проверки используют стандартную библиотеку Python.
Из корня `lab2`:
```powershell
python scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
Invoke-RestMethod http://127.0.0.1:8080/health/ready
$env:BASE_URL = 'http://127.0.0.1:8080'
python scripts/check_contract.py
python tests/smoke.py --base-url http://127.0.0.1:8080
python scripts/verify.py
```
`bootstrap.py` генерирует локальный случайный пароль PostgreSQL, URL подключения и постоянный ключ cursor в `secrets/`, создаёт `.env`. Повторный запуск сохраняет существующие значения. Эти файлы исключены из Git и Docker build context. Оператор должен сохранять `secrets/` между запусками с существующими volumes: новый пароль в файле не меняет пароль уже инициализированной PostgreSQL.
Compose дожидается PostgreSQL, запускает одноразовый `migrate`, после успешного завершения — API. Создаётся `schema_migrations` и атомарно применяется `migrations/001_initial.sql`. Повторный запуск сверяет SHA-256 применённых миграций и ничего не дублирует:
```powershell
docker compose run --rm --no-deps migrate
docker compose run --rm --no-deps migrate
```
В Linux/macOS после запуска стенда доступны Make-эквиваленты:
```sh
make check
make test BASE_URL=http://127.0.0.1:8080
make test-own
chmod +x scripts/restart.sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
make verify
```
В этой Windows-среде `check`/`test` выполнялись соответствующими Python-командами; persistence использовал `restart.cmd`, вызывающий тот же `scripts/restart.py`, что и `restart.sh`. Публичные проверки не изменены. Полная валидация OpenAPI отдельно: `make validate` после установки `requirements-dev.txt`; это проверка контракта, а не самого API.
Безопасная остановка: `docker compose down`. Следующий `docker compose up -d --wait` использует прежние volumes. **Намеренное удаление всех учебных данных:** `docker compose down --volumes` — уничтожает пользователей, чаты, сообщения и сессии этого проекта. Для persistence так останавливать нельзя. Скрипты проверки не удаляют основные volumes.
Демонстрационный сценарий API: `python scripts/demo.py`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Клиент → HTTP API → PostgreSQL и Redis. Проект таблиц, ограничений и транзакционных границ перед реализацией SQL: [infra/SCHEMA.md](infra/SCHEMA.md).
**PostgreSQL** — источник истины пользователей, password KDF, чатов, членства и сообщений. Есть UUID PK, UNIQUE username, FK, CHECK длины/формата. Составной FK сообщения на членство запрещает отправителя вне чата. Создание чата и всех участников — одна транзакция: FK violation отменяет весь результат. HTTP 201 регистрации/чата/сообщения выдаётся только после успешного COMMIT. Пользовательские значения передаются SQL-параметрами `%s`, не включаются в текст SQL.
**Redis** — единственное хранилище сессий, не кэш истории. Ключ `session:<sha256(token)>`, значение UUID пользователя. Токен имеет 256 бит случайности (`secrets.token_urlsafe(32)`), в исходном виде на сервере не сохраняется. `SET ... PXAT` назначает абсолютный expiry; чтение не продлевает TTL. Logout — DEL; expiry удаляет ключ автоматически. AOF включён с `appendfsync always`, `/data` находится на именованном volume `redisdata`. `maxmemory=64mb`, `noeviction`: при нехватке памяти вход завершится 503 вместо вытеснения действующих сессий. Redis не публикует порт на host; отсутствие Redis-аутентификации — учебное упрощение изолированной сети.
Пароль обрабатывается scrypt: N=16384, r=8, p=1, соль 16 bytes, dklen 32 bytes. Поддерживается 12–128 Unicode code points, включая двоеточие, без усечения по 72 bytes. Одинаковые пароли имеют разные salted hashes. Basic только для создания сессии, бизнес-endpoint требуют Bearer. `X-User-Id` игнорируется. Чужой и отсутствующий чат дают одинаковый 404 после аутентификации.
История не кэшируется: следующая операция чтения обращается к primary PostgreSQL и видит завершённый COMMIT. Индекс `messages_history(chat_id, created_at, id)` обслуживает keyset-пагинацию. Cursor содержит последнюю пару `(created_at,id)`, связан с чатом/пользователем и подписан HMAC постоянным ключом. Запрашивается `limit+1` строк; `next_cursor=null`, если продолжения нет. При конкурентных транзакциях поздний COMMIT с timestamp до уже пройденной позиции может быть пропущен обходом. Snapshot между HTTP-запросами и глобальный монотонный порядок COMMIT не обещаются; неизменная история обходится без потерь/повторов.
| Ресурс | Ограничение |
| --- | --- |
| HTTP workers / очередь | 16 / 32; переполнение → 503 |
| HTTP socket / JSON body | 2 s / 65 536 bytes |
| PostgreSQL pool | min 1, max 8, максимум 16 ожидающих, ожидание 1 s |
| PostgreSQL connect / statement / lock | 2 s / 2 s / 1 s; idle transaction 5 s |
| Redis pool | max 8, connect/read по 0.5 s, без автоматических retry |
| DNS внутри API | timeout 1 s, attempts 1; учитывает остановленный Docker service |
`/health/live` проверяет процесс. `/health/ready` проверяет PostgreSQL и Redis: обе зависимости критичны для полного синхронного пути. При отказе возвращается JSON 503 с request_id/Retry-After, без SQL, credentials и stack trace. Анонимный пользователь не становится доверенным. Регистрация технически может работать без Redis, но полный API в этом состоянии неготов.
API работает от UID 10001. Логи содержат method/status/request_id/latency без текста сообщений и заголовков авторизации. SIGTERM прекращает приём, завершает ограниченные workers и закрывает пулы. PostgreSQL/Redis используют именованные volumes; API локального состояния не имеет. Из lab1 перенесён обязательный REST API, заменены in-memory структуры и X-User-Id; исследования уровня 5 не переносились.
### Соответствие .env.example
| Переменная | Реальное использование |
| --- | --- |
| APP_HOST | Bind внутри API, по умолчанию 0.0.0.0 |
| APP_PORT | Публикуемый host port; внутри контейнера всегда 8080 |
| APP_ENV | Маркер учебного окружения, не меняет хранение |
| SESSION_TTL_SECONDS | TTL новых сессий, default 3600; для smoke/persistence ≥300 |
| SESSION_STORE | В этом решении redis, Compose фиксирует выбор |
| DATABASE_URL_FILE | `/run/secrets/database_url` из локального secrets/database_url |
| CACHE_URL_FILE | `/run/secrets/cache_url` из secrets/cache_url |
| CURSOR_KEY_FILE | `/run/secrets/cursor_key` из secrets/cursor_key |
Пути secrets в Compose фиксированы и совпадают с примером; значения URL не публикуются в tracked `.env.example`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Полный прогон: `python scripts/verify.py`. Все команды из корня `lab2`, при работающем стенде.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда/сценарий | Наблюдение | Доказательство |
| --- | --- | --- | --- |
| Структура контракта | `python scripts/check_contract.py` | PASS; публичные tests/contracts сохранены | [check.txt](evidence/check.txt) |
| Полная валидация OpenAPI | `python scripts/validate_openapi.py` с requirements-dev | PASS: OpenAPI 3.1 и JSON Schema | [validate.txt](evidence/validate.txt) |
| Запуск из чистой копии с нуля | `python scripts/clean_check.py` | Локальный git clone без .env/secrets, no-cache build, новые volumes, smoke и 3 проверки БД OK | [clean-clone.txt](evidence/clean-clone.txt) |
| Smoke и минимум lab1 с новым auth | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS: lab2 public API scenario | [smoke.txt](evidence/smoke.txt) |
| PostgreSQL, FK, UNIQUE, атомарность чата | `docker compose exec -T api python student-tests/test_database.py` | Ошибка второго участника: числа chats/chat_members не изменились | [database.txt](evidence/database.txt), [SQL](migrations/001_initial.sql) |
| Миграции с нуля / повторно | `docker compose up -d --build --wait`; test_repeat_migrations | Первичная миграция; два повтора Already applied, количества сохранены | [database.txt](evidence/database.txt) |
| Сохранность данных и сессий | Публичный resilience с restart.cmd / restart.sh | PASS, прежний токен читает прежнее сообщение; все 3 процесса перезапущены | [persistence.txt](evidence/persistence.txt), [restart.json](evidence/restart.json) |
| Basic/Bearer/logout/X-User-Id | Smoke, собственные API-тесты | Неверный пароль/Basic на business/X-User-Id → 401; logout 204, затем 401 | [smoke.txt](evidence/smoke.txt), [own-api.txt](evidence/own-api.txt) |
| Password KDF | test_password_salted_kdf; test_unicode_password_and_logout | Разные scrypt hashes одинаковых паролей; пароль 128 Unicode code points работает | [database.txt](evidence/database.txt), [own-api.txt](evidence/own-api.txt) |
| Единый запуск / volumes / secrets вне Git | `docker compose --env-file .env config --quiet` | Конфигурация успешна; 3 сервиса и одноразовая миграция | [compose.txt](evidence/compose.txt), [compose.yaml](compose.yaml) |
| Redis с TTL и persistence | `python scripts/check_expiry.py`; persistence | TTL=2: сначала 200, через 2.2 s → 401; обычная сессия пережила restart | [expiry.txt](evidence/expiry.txt), [redis.conf](infra/redis.conf) |
| Настройки durability | SHOW fsync/synchronous_commit/full_page_writes; Redis CONFIG GET | PostgreSQL: on/on/on; Redis: appendonly=yes, appendfsync=always | [PostgreSQL](evidence/durability-postgres.txt), [Redis](evidence/durability-redis.txt) |
| SQL parameters / bounded pools / timeouts | Реальный HTTP-путь, отказ зависимостей | 503 в пределах бюджета вместо зависания | [outages.txt](evidence/outages.txt), [core.py](src/messenger/core.py) |
| Гонка username | test_registration_race, 10 клиентов | Ровно 1×201 и 9×409 | [own-api.txt](evidence/own-api.txt) |
| EXPLAIN ANALYZE BUFFERS на ≥10 000 | `docker compose exec -T api python scripts/explain_history.py` | Index Scan messages_history, 51 строка, 4 shared hit blocks, без Seq Scan | [explain.json](evidence/explain.json) |
| Автоматические отказы Redis/БД | `python student-tests/test_outages.py` | По 3×503; live 200, ready 503; старые сессия/сообщение сохранены после восстановления | [outages.txt](evidence/outages.txt) |
| Собственные негативные/конкурентные тесты | `python student-tests/test_api.py` | 4 теста OK; 100 уникальных сообщений от 10 клиентов и обход страниц | [own-api.txt](evidence/own-api.txt) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Проверялась корректность уровня 4: durable stores сохраняют подтверждённое сообщение/сессию при перезапуске, а потеря зависимости ограниченно блокирует защищённые операции. Это не исследовательский трек 5 и не измерение максимальной нагрузки.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
Среда: Windows 10 build 19045, Docker Desktop Engine 29.8.2, Linux/WSL2 x86_64, 8 логических CPU и около 7.71 GiB RAM для Docker. Лимиты: API 2 CPU/512 MiB, PostgreSQL 2 CPU/512 MiB, Redis 1 CPU/128 MiB. Клиент Python 3.14.3, runtime 3.13.7. Digest хранилищ и сведения о машине: [environment.json](evidence/environment.json).
Профиль: последовательный smoke; гонка регистрации 10 клиентов; 100 POST сообщений от 10 клиентов; expiry; по три последовательных чтения при отказе зависимости; один план на новом чате с 10 000 коротких сообщений, cursor после позиции 9000, limit=50. Прогрев — создание fixtures и ANALYZE; план измерен на уже прогретых страницах. Повторяемость обеспечена скриптами. Статистические серии throughput не проводились, поэтому RPS/p50/p95/p99 не заявляются. [resources.txt](evidence/resources.txt) — единичный снимок CPU/RAM после проверок, не среднее/пик под нагрузкой.
| Воздействие | Ожидаем | Наблюдали | Восстановление / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Restart api/db/redis | Старый токен и сообщение действуют после ready | Persistence PASS; StartedAt всех 3 изменились, volumes прежние | В пределах лимитов публичного сценария; потерь нет |
| TTL=2 в отдельном API | После expiry → 401 | До expiry 200, через 2.2 s — 401 | Основной TTL=3600 не менялся |
| Logout | Токен недействителен | 204, затем 401 | Новый вход создаёт новую сессию |
| Redis stop | 503; анонимный вход запрещён | 3 ошибки за 1.005–1.027 s; X-User-Id без сессии → 401 | 0.604 s от начала start до ready/me; сессия/сообщение сохранены |
| PostgreSQL stop | 503; процесс жив | 3 ошибки за 1.005–1.008 s; live 200, ready 503 | 4.599 s от начала start до ready/me; сессия/сообщение сохранены |
| Неизвестный участник после INSERT chats | Полный rollback | Числа chats/chat_members до/после равны | Частичного чата нет |
SQL Execution Time **0.049 ms**, Index Scan отдал 51 строку, Shared Hit Blocks=4, Shared Read Blocks=0. Это один локальный прогретый SQL-запрос без HTTP/аутентификации: подтверждение использования индекса, не SLA. Не сравнивались альтернативные индексы/100 000 сообщений/backup — это задания оценки 5.
Первый сценарий отказа выявил DNS-ожидание остановленного контейнера дольше socket timeout. В итоговый Compose добавлены `dns_opt: timeout:1, attempts:1`; результаты выше получены после исправления.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один host, по одному PostgreSQL/Redis: проверяемый restart выдерживается, уничтожение диска и HA не покрыты. При отказе хранилища бизнес-путь временно недоступен.
- FLUSHDB/удаление Redis volume/повреждение AOF теряют сессии; нужно войти заново. PostgreSQL volume содержит бизнес-данные. Потеря cursor_key аннулирует прежние cursors.
- После COMMIT может оборваться транспорт: клиент не получит 201, хотя сообщение записано. Идемпотентность появляется в lab3 и здесь не реализована.
- Уже аутентифицированный конкурентный запрос может завершиться после logout; последующие проверки токена получают 401.
- HTTP опубликован только на 127.0.0.1, данные синтетические. Для удалённого использования нужны TLS и усиление конфигурации. В учебной БД используется роль, созданная официальным образом; production role management не реализован.
- Snapshot между страницами и упорядочивание конкурентных COMMIT не обещаются.
- Не реализованы треки 5: исследование согласованности кэша/stampede с измерениями, сравнение индексов на ≥100 000 и backup/restore.
## Источники
- PostgreSQL: [транзакции](https://www.postgresql.org/docs/17/tutorial-transactions.html), [WAL durability](https://www.postgresql.org/docs/17/wal-reliability.html), [EXPLAIN](https://www.postgresql.org/docs/17/using-explain.html).
- Redis: [persistence и AOF](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [PEXPIREAT](https://redis.io/docs/latest/commands/pexpireat/).
- Psycopg: [пулы](https://www.psycopg.org/psycopg3/docs/advanced/pool.html), [SQL parameters](https://www.psycopg.org/psycopg3/docs/basic/params.html).
- Python: [hashlib.scrypt](https://docs.python.org/3.13/library/hashlib.html#hashlib.scrypt), [secrets](https://docs.python.org/3.13/library/secrets.html).
- [RFC 7617 — Basic](https://www.rfc-editor.org/rfc/rfc7617.html), контракт и публичные проверки этого репозитория, собственная lab1, помощь OpenAI Codex, раскрытая в начале отчёта.
+67
View File
@@ -0,0 +1,67 @@
name: mtusi-lab2-osokin
x-app: &app
image: mtusi-messenger:lab2
build: .
environment: &app-env
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: 8080
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: redis
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_KEY_FILE: /run/secrets/cursor_key
secrets: [database_url, cache_url, cursor_key]
mem_limit: 512m
cpus: 2
dns_opt: ['timeout:1', 'attempts:1']
services:
db:
image: postgres:17.6-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
secrets: [db_password]
volumes: [pgdata:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 2
redis:
image: redis:7.4.5-bookworm
command: [redis-server, /usr/local/etc/redis/redis.conf]
volumes:
- redisdata:/data
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 128m
cpus: 1
migrate:
<<: *app
command: [python, -m, messenger.migrate]
healthcheck: {disable: true}
depends_on:
db: {condition: service_healthy}
api:
<<: *app
ports: ['127.0.0.1:${APP_PORT:-8080}:8080']
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
stop_grace_period: 8s
volumes:
pgdata:
redisdata:
secrets:
db_password: {file: ./secrets/db_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
cursor_key: {file: ./secrets/cursor_key}
+2
View File
@@ -0,0 +1,2 @@
$ python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+100
View File
@@ -0,0 +1,100 @@
$ git clone --no-hardlinks <workspace> <workspace>\output\clean-clone
Cloning into 'C:\Users\blobb\OneDrive\Документи\ChatGPT\высоконагружноут\lab2\output\clean-clone'...
done.
$ python scripts/bootstrap.py
Local .env and secrets ready (existing values preserved).
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env build --no-cache --quiet
Image mtusi-messenger:lab2 Building
Image mtusi-messenger:lab2 Building
Image mtusi-messenger:lab2 Built
Image mtusi-messenger:lab2 Built
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env up -d --wait
Volume mtusi-lab2-osokin-clean_pgdata Creating
Network mtusi-lab2-osokin-clean_default Creating
Volume mtusi-lab2-osokin-clean_redisdata Creating
Volume mtusi-lab2-osokin-clean_pgdata Creating
Volume mtusi-lab2-osokin-clean_redisdata Creating
Network mtusi-lab2-osokin-clean_default Creating
Volume mtusi-lab2-osokin-clean_pgdata Created
Volume mtusi-lab2-osokin-clean_pgdata Created
Volume mtusi-lab2-osokin-clean_redisdata Created
Volume mtusi-lab2-osokin-clean_redisdata Created
Network mtusi-lab2-osokin-clean_default Created
Network mtusi-lab2-osokin-clean_default Created
Container mtusi-lab2-osokin-clean-redis-1 Creating
Container mtusi-lab2-osokin-clean-db-1 Creating
Container mtusi-lab2-osokin-clean-redis-1 Created
Container mtusi-lab2-osokin-clean-db-1 Created
Container mtusi-lab2-osokin-clean-migrate-1 Creating
Container mtusi-lab2-osokin-clean-migrate-1 Created
Container mtusi-lab2-osokin-clean-api-1 Creating
Container mtusi-lab2-osokin-clean-api-1 Created
Container mtusi-lab2-osokin-clean-redis-1 Starting
Container mtusi-lab2-osokin-clean-db-1 Starting
Container mtusi-lab2-osokin-clean-redis-1 Started
Container mtusi-lab2-osokin-clean-db-1 Started
Container mtusi-lab2-osokin-clean-db-1 Waiting
Container mtusi-lab2-osokin-clean-db-1 Healthy
Container mtusi-lab2-osokin-clean-migrate-1 Starting
Container mtusi-lab2-osokin-clean-migrate-1 Started
Container mtusi-lab2-osokin-clean-redis-1 Waiting
Container mtusi-lab2-osokin-clean-migrate-1 Waiting
Container mtusi-lab2-osokin-clean-redis-1 Healthy
Container mtusi-lab2-osokin-clean-migrate-1 Exited
Container mtusi-lab2-osokin-clean-api-1 Starting
Container mtusi-lab2-osokin-clean-api-1 Started
Container mtusi-lab2-osokin-clean-redis-1 Waiting
Container mtusi-lab2-osokin-clean-migrate-1 Waiting
Container mtusi-lab2-osokin-clean-api-1 Waiting
Container mtusi-lab2-osokin-clean-db-1 Waiting
Container mtusi-lab2-osokin-clean-redis-1 Healthy
Container mtusi-lab2-osokin-clean-db-1 Healthy
Container mtusi-lab2-osokin-clean-migrate-1 Exited
Container mtusi-lab2-osokin-clean-api-1 Healthy
$ python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ python tests/smoke.py --base-url http://127.0.0.1:18082
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env exec -T api python student-tests/test_database.py
test_password_salted_kdf (__main__.Database.test_password_salted_kdf) ... ok
test_repeat_migrations (__main__.Database.test_repeat_migrations) ... Already applied: 001_initial.sql
Migrations committed.
Already applied: 001_initial.sql
Migrations committed.
ok
test_rollback_after_chat_insert (__main__.Database.test_rollback_after_chat_insert) ... ok
----------------------------------------------------------------------
Ran 3 tests in 2.993s
OK
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env logs --no-log-prefix migrate
Applied: 001_initial.sql
Migrations committed.
$ docker compose -p mtusi-lab2-osokin-clean --env-file .env down
Container mtusi-lab2-osokin-clean-api-1 Stopping
Container mtusi-lab2-osokin-clean-api-1 Stopped
Container mtusi-lab2-osokin-clean-api-1 Removing
Container mtusi-lab2-osokin-clean-api-1 Removed
Container mtusi-lab2-osokin-clean-migrate-1 Stopping
Container mtusi-lab2-osokin-clean-redis-1 Stopping
Container mtusi-lab2-osokin-clean-migrate-1 Stopped
Container mtusi-lab2-osokin-clean-migrate-1 Removing
Container mtusi-lab2-osokin-clean-migrate-1 Removed
Container mtusi-lab2-osokin-clean-db-1 Stopping
Container mtusi-lab2-osokin-clean-redis-1 Stopped
Container mtusi-lab2-osokin-clean-redis-1 Removing
Container mtusi-lab2-osokin-clean-redis-1 Removed
Container mtusi-lab2-osokin-clean-db-1 Stopped
Container mtusi-lab2-osokin-clean-db-1 Removing
Container mtusi-lab2-osokin-clean-db-1 Removed
Network mtusi-lab2-osokin-clean_default Removing
Network mtusi-lab2-osokin-clean_default Removed
+1
View File
@@ -0,0 +1 @@
$ docker compose --env-file .env config --quiet
+13
View File
@@ -0,0 +1,13 @@
$ docker compose exec -T api python student-tests/test_database.py
test_password_salted_kdf (__main__.Database.test_password_salted_kdf) ... ok
test_repeat_migrations (__main__.Database.test_repeat_migrations) ... Already applied: 001_initial.sql
Migrations committed.
Already applied: 001_initial.sql
Migrations committed.
ok
test_rollback_after_chat_insert (__main__.Database.test_rollback_after_chat_insert) ... ok
----------------------------------------------------------------------
Ran 3 tests in 2.310s
OK
+8
View File
@@ -0,0 +1,8 @@
$ python scripts/demo.py
1. Readiness 200; registered three users; Basic login returned Bearer sessions.
2. Created chat: c2817969-606b-4551-bff4-c98ba084c1b4 ; members: Alice and Bob.
3. POST message -> 201; message id: ac5d70bf-4bd4-4da8-a17b-e8505600bc15
4. Bob reads Alice's committed message -> 200: Привет, Боб! Это лабораторная 2.
5. Eve is authenticated but not a chat member -> 404.
6. X-User-Id without Bearer -> 401.
7. Logout -> 204; previous token -> 401. Demo PASS.
+15
View File
@@ -0,0 +1,15 @@
$ docker compose exec -T db psql -U messenger -d messenger -c SHOW fsync; -c SHOW synchronous_commit; -c SHOW full_page_writes;
fsync
-------
on
(1 row)
synchronous_commit
--------------------
on
(1 row)
full_page_writes
------------------
on
(1 row)
+5
View File
@@ -0,0 +1,5 @@
$ docker compose exec -T redis redis-cli CONFIG GET appendonly appendfsync
appendfsync
always
appendonly
yes
+22
View File
@@ -0,0 +1,22 @@
{
"verified_at_utc": "2026-10-26T12:25:05.273386+00:00",
"os": "Windows-10-10.0.19045-SP0",
"client_python": "3.14.3",
"docker": {
"ServerVersion": "29.8.2",
"OperatingSystem": "Docker Desktop",
"KernelVersion": "6.18.40.1-microsoft-standard-WSL2",
"NCPU": 8,
"MemTotal": 8275755008,
"Architecture": "x86_64"
},
"images": {
"postgres:17.6-bookworm": [
"postgres@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3"
],
"redis:7.4.5-bookworm": [
"redis@sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb"
]
},
"runtime_python": "Python 3.13.7"
}
+4
View File
@@ -0,0 +1,4 @@
$ python scripts/check_expiry.py
Container mtusi-lab2-osokin-api-run-caf7bb185347 Creating
Container mtusi-lab2-osokin-api-run-caf7bb185347 Created
PASS: session accepted before expiry; rejected after 2.2s with SESSION_TTL_SECONDS=2.
+81
View File
@@ -0,0 +1,81 @@
{
"messages": 10000,
"cursor_position": 9000,
"page_limit": 50,
"fetched_with_lookahead": 51,
"sql": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s",
"explain": [
{
"Plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.29,
"Total Cost": 17.77,
"Plan Rows": 51,
"Plan Width": 68,
"Actual Startup Time": 0.008,
"Actual Total Time": 0.025,
"Actual Rows": 51,
"Actual Loops": 1,
"Shared Hit Blocks": 4,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_history",
"Relation Name": "messages",
"Alias": "messages",
"Startup Cost": 0.29,
"Total Cost": 473.36,
"Plan Rows": 1380,
"Plan Width": 68,
"Actual Startup Time": 0.007,
"Actual Total Time": 0.021,
"Actual Rows": 51,
"Actual Loops": 1,
"Index Cond": "((chat_id = '137c9dcf-9f70-465c-862c-595c9a50d5c3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.009+00'::timestamp with time zone, '3873da86-1a3e-2129-c84e-1daf5c084a0d'::uuid)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 4,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
},
"Planning": {
"Shared Hit Blocks": 6,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
},
"Planning Time": 0.079,
"Triggers": [],
"Execution Time": 0.049
}
]
}
+39
View File
@@ -0,0 +1,39 @@
$ python student-tests/test_outages.py
Container mtusi-lab2-osokin-redis-1 Stopping
Container mtusi-lab2-osokin-redis-1 Stopped
Container mtusi-lab2-osokin-redis-1 Starting
Container mtusi-lab2-osokin-redis-1 Started
Container mtusi-lab2-osokin-db-1 Stopping
Container mtusi-lab2-osokin-db-1 Stopped
Container mtusi-lab2-osokin-db-1 Starting
Container mtusi-lab2-osokin-db-1 Started
[
{
"service": "redis",
"errors": 3,
"status": 503,
"seconds": [
1.0267414001282305,
1.0045876000076532,
1.007952299900353
],
"recovery_seconds": 0.604,
"original_session_preserved": true,
"original_message_preserved": true,
"anonymous_status": 401
},
{
"service": "db",
"errors": 3,
"status": 503,
"seconds": [
1.0076619999017566,
1.0045177999418229,
1.0052553000859916
],
"recovery_seconds": 4.599,
"original_session_preserved": true,
"original_message_preserved": true,
"anonymous_status": 401
}
]
+10
View File
@@ -0,0 +1,10 @@
$ python student-tests/test_api.py
test_concurrent_messages (__main__.API.test_concurrent_messages) ... ok
test_registration_race (__main__.API.test_registration_race) ... ok
test_unicode_password_and_logout (__main__.API.test_unicode_password_and_logout) ... ok
test_validation_access_and_pagination (__main__.API.test_validation_access_and_pagination) ... ok
----------------------------------------------------------------------
Ran 4 tests in 4.016s
OK
+3
View File
@@ -0,0 +1,3 @@
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+4
View File
@@ -0,0 +1,4 @@
$ docker stats --no-stream --format {{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}} mtusi-lab2-osokin-api-1 mtusi-lab2-osokin-db-1 mtusi-lab2-osokin-redis-1
mtusi-lab2-osokin-api-1 CPU=51.26% RAM=158.7MiB / 512MiB
mtusi-lab2-osokin-db-1 CPU=8.94% RAM=27.76MiB / 512MiB
mtusi-lab2-osokin-redis-1 CPU=0.44% RAM=4.301MiB / 128MiB
+56
View File
@@ -0,0 +1,56 @@
{
"before": {
"api": {
"id": "b6b4e6f8c09ee890165395f2af5e40945dd74ce270a9c35c09b946db8ed160ef",
"started_at": "2026-10-26T20:13:56.437921569Z",
"mounts": []
},
"db": {
"id": "9a25c8ce8dd3a0a64d4def2f27765f792fb00739774c0e0926fe0cb1017c27fd",
"started_at": "2026-10-26T20:13:49.316227457Z",
"mounts": [
{
"name": "lab2-defense-0eaa9a8d_pgdata",
"destination": "/var/lib/postgresql/data"
}
]
},
"redis": {
"id": "8009d787a3e25c88d024c121ff5dd00a613e77b785a30fbe4b08b1dcb57a021e",
"started_at": "2026-10-26T20:13:49.903677339Z",
"mounts": [
{
"name": "lab2-defense-0eaa9a8d_redisdata",
"destination": "/data"
}
]
}
},
"after": {
"api": {
"id": "b6b4e6f8c09ee890165395f2af5e40945dd74ce270a9c35c09b946db8ed160ef",
"started_at": "2026-10-26T20:15:24.833818504Z",
"mounts": []
},
"db": {
"id": "9a25c8ce8dd3a0a64d4def2f27765f792fb00739774c0e0926fe0cb1017c27fd",
"started_at": "2026-10-26T20:15:24.683887137Z",
"mounts": [
{
"name": "lab2-defense-0eaa9a8d_pgdata",
"destination": "/var/lib/postgresql/data"
}
]
},
"redis": {
"id": "8009d787a3e25c88d024c121ff5dd00a613e77b785a30fbe4b08b1dcb57a021e",
"started_at": "2026-10-26T20:15:24.575559122Z",
"mounts": [
{
"name": "lab2-defense-0eaa9a8d_redisdata",
"destination": "/data"
}
]
}
}
}
+2
View File
@@ -0,0 +1,2 @@
$ python tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
+2
View File
@@ -0,0 +1,2 @@
$ ../lab1/.venv/Scripts/python.exe scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+17
View File
@@ -0,0 +1,17 @@
# Схема и транзакции (проект перед реализацией SQL)
Источник истины бизнес-данных — PostgreSQL; единственное хранилище сессий — Redis.
* users: UUID PK, username UNIQUE с CHECK формата, display_name, password_hash (scrypt), created_at timestamptz.
* chats: UUID PK, title, created_at timestamptz.
* chat_members: (chat_id, user_id) PK, FK к chats и users; position для сохранения порядка участников.
* messages: UUID PK, FK chat_id → chats; составной FK (chat_id, sender_id) → chat_members.
* schema_migrations: имя миграции PK, SHA-256 содержимого, applied_at. DDL и запись версии в одной транзакции под advisory lock.
Индексы: UNIQUE username защищает конкурентную регистрацию. messages_history(chat_id, created_at, id) обслуживает `WHERE chat_id = $1 AND (created_at,id) > ($2,$3) ORDER BY created_at,id LIMIT $4`, без OFFSET. chat_members_user(user_id,chat_id) — список доступных чатов. chats_order(created_at,id) — порядок чатов.
Создание чата: INSERT chats + INSERT всех участников в одной транзакции; неизвестный участник вызывает FK violation и rollback всего чата. Регистрация и отправка сообщения — отдельные транзакции; HTTP 201 только после успешного COMMIT. Чтения — READ COMMITTED с primary PostgreSQL; история не кэшируется.
Redis: `session:<sha256(token)>` → UUID пользователя, абсолютный expiry (PXAT); TTL не обновляется при чтении. Logout — DEL. AOF + appendfsync always, именованный volume. Потеря Redis: 503, никакого доверия X-User-Id; restart сохраняет действующие сессии, FLUSHDB отзывает все. Истечение/DEL не требует инвалидации других кэшей.
Cursor подписан HMAC-SHA256 постоянным локальным ключом, связан с chat_id либо user_id; хранит (created_at,id). Это позиция, не snapshot: конкурентная поздно закоммиченная запись со старым timestamp может оказаться за уже пройденным cursor. Неизменная история обходится без повторов; UUID разрешает равные timestamps. Часы назначает PostgreSQL, монотонность между транзакциями не обещается.
+10
View File
@@ -0,0 +1,10 @@
bind 0.0.0.0
protected-mode no
port 6379
dir /data
appendonly yes
appendfsync always
save ""
maxmemory 64mb
maxmemory-policy noeviction
# Only the private Compose network can reach Redis; no host port is published.
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX chat_members_user ON chat_members(user_id, chat_id);
CREATE INDEX chats_order ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history ON messages(chat_id, created_at, id);
+5
View File
@@ -0,0 +1,5 @@
psycopg==3.2.10
psycopg-binary==3.2.10
psycopg-pool==3.2.6
redis==6.4.0
typing_extensions==4.15.0
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once; never print them or replace existing files."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
folder = root / 'secrets'
folder.mkdir(exist_ok=True)
password_file = folder / 'db_password'
if not password_file.exists():
password_file.write_text(secrets.token_hex(24), encoding='utf-8')
password = password_file.read_text(encoding='utf-8').strip()
values = {
'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger',
'cache_url': 'redis://redis:6379/0',
'cursor_key': secrets.token_hex(32),
}
for name, value in values.items():
target = folder / name
if not target.exists():
target.write_text(value, encoding='utf-8')
if not (root / '.env').exists():
(root / '.env').write_text('APP_HOST=0.0.0.0\nAPP_PORT=8080\nAPP_ENV=lab\nSESSION_TTL_SECONDS=3600\nSESSION_STORE=redis\n', encoding='utf-8')
print('Local .env and secrets ready (existing values preserved).')
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env python3
"""Separate API container with 2 second TTL; main API configuration is unchanged."""
from pathlib import Path
import subprocess
import sys
root = Path(__file__).resolve().parents[1]
command = ['docker', 'compose', 'run', '-d', '--no-deps', '-e', 'SESSION_TTL_SECONDS=2', 'api']
identifier = subprocess.check_output(command, cwd=root, text=True).strip()
try:
# Start the client in the temporary container after its HTTP server is ready.
subprocess.run(['docker', 'exec', identifier, 'python', '-c',
"import time,urllib.request; "
"\nfor _ in range(50):\n try:\n urllib.request.urlopen('http://localhost:8080/health/ready',timeout=3); break"
"\n except OSError: time.sleep(.2)\nelse: raise RuntimeError('API not ready')"], check=True)
subprocess.run(['docker', 'exec', identifier, 'python', 'student-tests/test_expiry.py'], check=True)
finally:
subprocess.run(['docker', 'rm', '-f', identifier], check=True, stdout=subprocess.DEVNULL)
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Check committed sources in a new local clone with fresh secrets and new volumes."""
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
clone = ROOT / 'output' / 'clean-clone'
project = 'mtusi-lab2-osokin-clean'
env = dict(os.environ, PYTHONIOENCODING='utf-8', BASE_URL='http://127.0.0.1:18082')
def main():
if clone.exists():
raise SystemExit('output/clean-clone already exists; preserve or move it before repeating this check.')
clone.parent.mkdir(exist_ok=True)
log = []
def run(command, cwd):
log.append('$ ' + ' '.join(command).replace(str(ROOT), '<workspace>').replace(sys.executable, 'python'))
process = subprocess.run(command, cwd=cwd, env=env, text=True, encoding='utf-8', errors='replace',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log.append(process.stdout)
cleaned = '\n'.join(line.rstrip() for line in '\n'.join(log).splitlines()) + '\n'
(ROOT / 'evidence' / 'clean-clone.txt').write_text(cleaned, encoding='utf-8')
if process.returncode:
raise SystemExit('Clean clone check failed; see evidence/clean-clone.txt')
run(['git', 'clone', '--no-hardlinks', str(ROOT), str(clone)], ROOT)
assert not (clone / '.env').exists()
assert not (clone / 'secrets').exists()
run([sys.executable, 'scripts/bootstrap.py'], clone)
local_env = clone / '.env'
local_env.write_text(local_env.read_text(encoding='utf-8').replace('APP_PORT=8080', 'APP_PORT=18082'), encoding='utf-8')
compose = ['docker', 'compose', '-p', project, '--env-file', '.env']
try:
run(compose + ['build', '--no-cache', '--quiet'], clone)
run(compose + ['up', '-d', '--wait'], clone)
run([sys.executable, 'scripts/check_contract.py'], clone)
run([sys.executable, 'tests/smoke.py', '--base-url', env['BASE_URL']], clone)
run(compose + ['exec', '-T', 'api', 'python', 'student-tests/test_database.py'], clone)
run(compose + ['logs', '--no-log-prefix', 'migrate'], clone)
finally:
# Keep even the disposable check's volumes; only stop its isolated containers.
run(compose + ['down'], clone)
print('PASS: clean local clone; no copied secrets, fresh volumes, no-cache build, smoke/database checks.')
if __name__ == '__main__':
main()
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
"""Human-readable demonstration. Passwords and session tokens stay in memory."""
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
c = Client()
alice, bob, eve, chat = c.bootstrap()
print('1. Readiness 200; registered three users; Basic login returned Bearer sessions.')
print('2. Created chat:', chat['id'], '; members: Alice and Bob.')
message = c.message(alice, chat, 'Привет, Боб! Это лабораторная 2.')['json']
print('3. POST message -> 201; message id:', message['id'])
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json']
assert page['items'] == [message]
print('4. Bob reads Alice\'s committed message -> 200:', message['text'])
c.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve)
print('5. Eve is authenticated but not a chat member -> 404.')
c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': alice['id']})
print('6. X-User-Id without Bearer -> 401.')
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
c.call('GET', '/api/v1/users/me', 401, actor=alice)
print('7. Logout -> 204; previous token -> 401. Demo PASS.')
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""One index, 10,000 messages; grade 4 evidence, not a grade 5 comparison."""
import json
import uuid
from messenger.core import Config, HISTORY_SQL, Store
store = Store(Config.from_env())
try:
actor = store.create_user(dict(username='plan_' + uuid.uuid4().hex[:20],
display_name='Plan fixture', password='synthetic-plan-password'))['id']
chat = store.create_chat(actor, dict(title='EXPLAIN 10000', member_ids=[actor]))
with store.pool.connection() as conn:
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) "
"SELECT md5(%s || i::text)::uuid, %s, %s, 'fixture ' || i, "
"timestamptz '2026-01-01 UTC' + i * interval '1 microsecond' "
"FROM generate_series(1,10000) i", (chat['id'], chat['id'], actor))
conn.execute('ANALYZE messages')
key = conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s '
'ORDER BY created_at,id LIMIT 1 OFFSET 8999', (chat['id'],)).fetchone()
plan = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + HISTORY_SQL,
(chat['id'], key['created_at'], key['id'], 51)).fetchone()['QUERY PLAN']
count = conn.execute('SELECT count(*) AS n FROM messages WHERE chat_id=%s', (chat['id'],)).fetchone()['n']
# OFFSET is used once to choose a fixture position, never in the API query.
assert count == 10000
nodes = []
def walk(node):
nodes.append(node)
for child in node.get('Plans', []):
walk(child)
walk(plan[0]['Plan'])
assert any(n.get('Index Name') == 'messages_history' for n in nodes)
assert all(n['Node Type'] != 'Seq Scan' for n in nodes)
print(json.dumps(dict(messages=count, cursor_position=9000, page_limit=50, fetched_with_lookahead=51,
sql=HISTORY_SQL, explain=plan), ensure_ascii=False, indent=2))
finally:
store.close()
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0.."
python scripts\restart.py
exit /b %errorlevel%
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Restart precisely this lab's API, PostgreSQL and Redis, keeping both volumes."""
import json
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def compose(*args):
return subprocess.check_output(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, text=True)
def states():
result = {}
for service in ('api', 'db', 'redis'):
identifier = compose('ps', '-q', service).strip()
data = json.loads(subprocess.check_output(['docker', 'inspect', identifier], text=True))[0]
result[service] = dict(id=identifier, started_at=data['State']['StartedAt'],
mounts=[dict(name=m.get('Name'), destination=m['Destination'])
for m in data['Mounts'] if m['Type'] == 'volume'])
return result
def main():
before = states()
subprocess.run(['docker', 'compose', '--env-file', '.env', 'restart', 'api', 'db', 'redis'],
cwd=ROOT, check=True)
after = states()
for service in before:
assert before[service]['started_at'] != after[service]['started_at'], service + ' did not restart'
assert before[service]['mounts'] == after[service]['mounts'], service + ' changed volumes'
(ROOT / 'evidence' / 'restart.json').write_text(json.dumps(dict(before=before, after=after), indent=2) + '\n')
print('Restarted api/db/redis; volume names unchanged. Evidence: evidence/restart.json')
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/restart.py
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Run the grade 4 checks and preserve small sanitized evidence files."""
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
from datetime import datetime, timezone
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / 'evidence'
env = dict(os.environ, BASE_URL=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'), PYTHONIOENCODING='utf-8')
python = sys.executable
if hasattr(sys.stdout, 'reconfigure'):
sys.stdout.reconfigure(encoding='utf-8')
def run(name, command):
print('Running', name, flush=True)
process = subprocess.run(command, cwd=ROOT, env=env, text=True, encoding='utf-8', errors='replace',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = '$ ' + ' '.join(command).replace(python, 'python') + '\n' + process.stdout
log = '\n'.join(line.rstrip() for line in log.splitlines()).rstrip() + '\n'
(EVIDENCE / (name + '.txt')).write_text(log, encoding='utf-8')
print(process.stdout, end='', flush=True)
if process.returncode:
raise SystemExit('Check failed: ' + name)
def main():
EVIDENCE.mkdir(exist_ok=True)
run('check', [python, 'scripts/check_contract.py'])
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
run('smoke', [python, 'tests/smoke.py', '--base-url', env['BASE_URL']])
run('own-api', [python, 'student-tests/test_api.py'])
run('database', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_database.py'])
run('persistence', [python, 'tests/resilience.py', '--mode', 'persistence', '--action-script',
'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh', '--base-url', env['BASE_URL']])
run('expiry', [python, 'scripts/check_expiry.py'])
run('outages', [python, 'student-tests/test_outages.py'])
run('demo', [python, 'scripts/demo.py'])
run('durability-postgres', ['docker', 'compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger',
'-d', 'messenger', '-c', 'SHOW fsync;', '-c', 'SHOW synchronous_commit;',
'-c', 'SHOW full_page_writes;'])
run('durability-redis', ['docker', 'compose', 'exec', '-T', 'redis', 'redis-cli',
'CONFIG', 'GET', 'appendonly', 'appendfsync'])
output = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python',
'scripts/explain_history.py'], cwd=ROOT, text=True)
plan = json.loads(output)
(EVIDENCE / 'explain.json').write_text(json.dumps(plan, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print('PASS: EXPLAIN on', plan['messages'], 'messages uses messages_history.', flush=True)
run('resources', ['docker', 'stats', '--no-stream', '--format',
'{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}',
'mtusi-lab2-osokin-api-1', 'mtusi-lab2-osokin-db-1', 'mtusi-lab2-osokin-redis-1'])
environment = dict(verified_at_utc=datetime.now(timezone.utc).isoformat(), os=platform.platform(),
client_python=platform.python_version())
environment['docker'] = json.loads(subprocess.check_output(
['docker', 'info', '--format', '{{json .}}'], text=True))
# Whitelist machine properties; never persist Docker client config/proxy details.
environment['docker'] = {k: environment['docker'][k] for k in
('ServerVersion', 'OperatingSystem', 'KernelVersion', 'NCPU', 'MemTotal', 'Architecture')}
environment['images'] = {}
environment['runtime_python'] = subprocess.check_output(
['docker', 'compose', 'exec', '-T', 'api', 'python', '--version'], cwd=ROOT, text=True).strip()
for image in ('postgres:17.6-bookworm', 'redis:7.4.5-bookworm'):
environment['images'][image] = json.loads(subprocess.check_output(
['docker', 'image', 'inspect', image, '--format', '{{json .RepoDigests}}'], text=True))
(EVIDENCE / 'environment.json').write_text(json.dumps(environment, indent=2) + '\n', encoding='utf-8')
print('PASS: all grade 4 scenarios completed; no grade 5 track was run.')
if __name__ == '__main__':
main()
+1
View File
@@ -0,0 +1 @@
"""Lab 1: bounded, single-process in-memory messenger."""
+301
View File
@@ -0,0 +1,301 @@
"""Validation and persistent state. HTTP 201 is emitted after transaction exit/COMMIT."""
import base64
from dataclasses import dataclass, fields
from datetime import datetime, timezone
import hashlib
import hmac
import json
import os
from pathlib import Path
import re
import secrets
import uuid
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
class APIError(Exception):
def __init__(self, status, message):
self.status, self.message = status, message
@dataclass
class Config:
app_host: str = '0.0.0.0'
app_port: int = 8080
workers: int = 16
request_queue: int = 32
socket_timeout: int = 2
max_body_bytes: int = 65536
session_ttl_seconds: int = 3600
@classmethod
def from_env(cls):
values = {}
for field in fields(cls):
raw = os.environ.get(field.name.upper(), field.default)
values[field.name] = str(raw) if field.type is str else int(raw)
if field.type is int and values[field.name] < 1:
raise ValueError(field.name + ' must be positive')
result = cls(**values)
if result.app_port > 65535 or result.workers > 32 or result.request_queue > 128:
raise ValueError('port/workers/queue outside bounds')
if result.max_body_bytes > 1048576 or result.socket_timeout > 5:
raise ValueError('body <= 1 MiB and socket timeout <= 5 required')
if os.getenv('SESSION_STORE', 'redis') != 'redis':
raise ValueError('This solution uses Redis sessions')
return result
def secret_file(name):
return Path(os.environ[name]).read_text(encoding='utf-8').strip()
def db_connect_kwargs():
return dict(connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1000 '
'-c idle_in_transaction_session_timeout=5000', row_factory=dict_row)
def uuid_value(value):
if not isinstance(value, str) or not re.fullmatch(
r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value):
raise APIError(400, 'Invalid UUID')
return str(uuid.UUID(value))
def string(value, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum or '\x00' in value:
raise APIError(400, 'Invalid string')
try:
value.encode('utf-8')
except UnicodeError:
raise APIError(400, 'Invalid Unicode') from None
return value
def object_fields(body, expected):
if not isinstance(body, dict) or set(body) != set(expected):
raise APIError(400, 'Invalid object fields')
def password_value(value):
if not isinstance(value, str) or not value.strip() or not 12 <= len(value) <= 128:
raise APIError(400, 'Invalid password length')
try:
value.encode('utf-8')
except UnicodeError:
raise APIError(400, 'Invalid Unicode') from None
return value
def password_hash(value, salt=None):
salt = salt or secrets.token_bytes(16)
digest = hashlib.scrypt(value.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32)
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
def password_matches(value, encoded):
# Only this versioned format is generated by registration.
salt = bytes.fromhex(encoded.split('$')[4])
return hmac.compare_digest(password_hash(value, salt), encoded)
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def public_row(row):
return {key: (str(value) if isinstance(value, uuid.UUID) else stamp(value)
if isinstance(value, datetime) else value)
for key, value in row.items() if key != 'password_hash'}
USER_COLUMNS = 'id, username, display_name, created_at'
MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at'
# Shared by the real history endpoint and EXPLAIN evidence; all values are parameters.
HISTORY_SQL = ('SELECT ' + MESSAGE_COLUMNS + ' FROM messages WHERE chat_id = %s '
'AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s')
class Store:
def __init__(self, config):
self.config = config
self.secret = bytes.fromhex(secret_file('CURSOR_KEY_FILE'))
if len(self.secret) < 32:
raise ValueError('Cursor key must be at least 256 bits')
self.pool = ConnectionPool(secret_file('DATABASE_URL_FILE'), min_size=1, max_size=8,
timeout=1, max_waiting=16, kwargs=db_connect_kwargs(),
check=ConnectionPool.check_connection, open=True)
self.cache_pool = redis.ConnectionPool.from_url(
secret_file('CACHE_URL_FILE'), max_connections=8, socket_connect_timeout=.5,
socket_timeout=.5, decode_responses=True, retry=Retry(NoBackoff(), 0))
self.cache = redis.Redis(connection_pool=self.cache_pool)
self.dummy_hash = password_hash('dummy-password-for-timing')
def close(self):
self.pool.close(timeout=2)
self.cache_pool.disconnect()
def ready(self):
with self.pool.connection() as conn:
conn.execute('SELECT 1')
self.cache.ping()
def create_user(self, body):
object_fields(body, ('username', 'display_name', 'password'))
username = string(body['username'], 32)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
raise APIError(400, 'Invalid username')
display = string(body['display_name'], 100)
hashed = password_hash(password_value(body['password']))
try:
with self.pool.connection() as conn:
row = conn.execute('INSERT INTO users(id, username, display_name, password_hash) '
'VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS,
(uuid.uuid4(), username, display, hashed)).fetchone()
except psycopg.errors.UniqueViolation:
raise APIError(409, 'Username already exists') from None
return public_row(row)
def login(self, authorization):
try:
scheme, encoded = authorization.split(' ', 1)
if scheme.lower() != 'basic' or len(encoded) > 1024:
raise ValueError()
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
raise ValueError()
password_value(password)
except (AttributeError, ValueError, UnicodeError, APIError):
raise APIError(401, 'Invalid credentials') from None
with self.pool.connection() as conn:
row = conn.execute('SELECT * FROM users WHERE username = %s', (username,)).fetchone()
matched = password_matches(password, row['password_hash'] if row else self.dummy_hash)
if not matched or row is None:
raise APIError(401, 'Invalid credentials')
token = secrets.token_urlsafe(32)
expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.config.session_ttl_seconds * 1000
self.cache.set(self.session_key(token), str(row['id']), pxat=expires_ms)
return dict(token=token, token_type='Bearer',
expires_at=stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)),
user=public_row(row))
@staticmethod
def session_key(token):
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
def auth(self, header):
try:
scheme, token = header.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError()
except (ValueError, AttributeError):
raise APIError(401, 'Invalid session') from None
key = self.session_key(token)
actor = self.cache.get(key)
if actor is None:
raise APIError(401, 'Invalid session')
return actor, key
def me(self, actor):
with self.pool.connection() as conn:
row = conn.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id = %s', (actor,)).fetchone()
if row is None:
raise APIError(401, 'Invalid session')
return public_row(row)
def chat_access(self, actor, identifier):
identifier = uuid_value(identifier)
with self.pool.connection() as conn:
row = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s',
(identifier, actor)).fetchone()
if row is None:
raise APIError(404, 'Object not found')
return identifier
@staticmethod
def chat_row(conn, row):
members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position',
(row['id'],)).fetchall()
return dict(public_row(row), member_ids=[str(x['user_id']) for x in members])
def chat(self, chat_id):
with self.pool.connection() as conn:
row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()
return self.chat_row(conn, row)
def create_chat(self, actor, body):
object_fields(body, ('title', 'member_ids'))
title = string(body['title'], 100)
members = body['member_ids']
if not isinstance(members, list) or not 1 <= len(members) <= 100:
raise APIError(400, 'Invalid members')
members = [uuid_value(x) for x in members]
if len(members) != len(set(members)) or actor not in members:
raise APIError(400, 'Invalid members')
try:
with self.pool.connection() as conn:
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *',
(uuid.uuid4(), title)).fetchone()
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)',
[(row['id'], member, pos) for pos, member in enumerate(members)])
except psycopg.errors.ForeignKeyViolation:
raise APIError(400, 'Invalid members') from None
return dict(public_row(row), member_ids=members)
def create_message(self, actor, chat_id, body):
object_fields(body, ('text',))
text = string(body['text'], 4000)
with self.pool.connection() as conn:
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
'RETURNING ' + MESSAGE_COLUMNS, (uuid.uuid4(), chat_id, actor, text)).fetchone()
return public_row(row)
def encode_cursor(self, scope, key):
payload = json.dumps([1, scope, *key], separators=(',', ':')).encode()
return base64.urlsafe_b64encode(payload + hmac.digest(self.secret, payload, 'sha256')).rstrip(b'=').decode()
def decode_cursor(self, cursor, scope):
try:
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', cursor):
raise ValueError()
raw = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_', validate=True)
if base64.urlsafe_b64encode(raw).rstrip(b'=').decode() != cursor:
raise ValueError()
payload, signature = raw[:-32], raw[-32:]
if not hmac.compare_digest(signature, hmac.digest(self.secret, payload, 'sha256')):
raise ValueError()
version, resource, timestamp, identifier = json.loads(payload)
if version != 1 or resource != scope or not isinstance(timestamp, str) or not timestamp.endswith('Z'):
raise ValueError()
return datetime.fromisoformat(timestamp.replace('Z', '+00:00')), uuid_value(identifier)
except (ValueError, TypeError, UnicodeError, APIError):
raise APIError(400, 'Invalid cursor') from None
def page(self, actor, chat_id, query):
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
raise APIError(400, 'Invalid query')
limit = query.get('limit', ['50'])[0]
if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100:
raise APIError(400, 'Invalid limit')
limit = int(limit)
scope = 'messages:' + chat_id if chat_id else 'chats:' + actor
key = self.decode_cursor(query['cursor'][0], scope) if 'cursor' in query else (
datetime.min.replace(tzinfo=timezone.utc), '00000000-0000-0000-0000-000000000000')
with self.pool.connection() as conn:
if chat_id:
rows = conn.execute(HISTORY_SQL, (chat_id, *key, limit + 1)).fetchall()
items = [public_row(row) for row in rows[:limit]]
else:
rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id '
'WHERE m.user_id=%s AND (c.created_at,c.id) > (%s,%s) '
'ORDER BY c.created_at,c.id LIMIT %s', (actor, *key, limit + 1)).fetchall()
items = [self.chat_row(conn, row) for row in rows[:limit]]
cursor = self.encode_cursor(scope, (items[-1]['created_at'], items[-1]['id'])) if len(rows) > limit else None
return dict(items=items, next_cursor=cursor)
+30
View File
@@ -0,0 +1,30 @@
"""Transactional, versioned migrations; repeat application is a no-op."""
import hashlib
from pathlib import Path
import psycopg
from .core import secret_file, db_connect_kwargs
def main():
with psycopg.connect(secret_file('DATABASE_URL_FILE'), **db_connect_kwargs()) as conn:
conn.execute('SELECT pg_advisory_xact_lock(2026, 2)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
'(name text PRIMARY KEY, sha256 text NOT NULL, '
'applied_at timestamptz NOT NULL DEFAULT clock_timestamp())')
for path in sorted((Path(__file__).resolve().parents[2] / 'migrations').glob('*.sql')):
content = path.read_bytes()
digest = hashlib.sha256(content).hexdigest()
row = conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s', (path.name,)).fetchone()
if row:
if row['sha256'] != digest:
raise RuntimeError('Applied migration checksum mismatch')
print('Already applied:', path.name)
continue
conn.execute(content.decode('utf-8'), prepare=False)
conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)', (path.name, digest))
print('Applied:', path.name)
print('Migrations committed.')
if __name__ == '__main__':
main()
+331
View File
@@ -0,0 +1,331 @@
"""Bounded HTTP worker pool; one request per connection; JSON logs omit bodies."""
from collections import Counter
from http.server import BaseHTTPRequestHandler, HTTPServer
import http.client
import json
import logging
import queue
import signal
import socket
import threading
import time
from urllib.parse import parse_qs, urlsplit
import uuid
import psycopg
from psycopg_pool import PoolTimeout, TooManyRequests
import redis
from .core import APIError, Config, Store
CODES = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict",
413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited",
503: "unavailable", 500: "internal_error"}
def error_body(status, message, request_id):
return {"error": {"code": CODES.get(status, "invalid_request"),
"message": message, "request_id": request_id}}
def log_event(**data):
logging.info(json.dumps(data, ensure_ascii=True))
class Server(HTTPServer):
allow_reuse_address = True
def __init__(self, config):
self.config, self.store = config, Store(config)
self.jobs = queue.Queue(maxsize=config.request_queue)
self.draining = threading.Event()
self.stats_lock = threading.Lock()
self.statuses = Counter()
self.active = 0
self.clients = set()
self.started = time.monotonic()
self.request_queue_size = config.request_queue
super().__init__((config.app_host, config.app_port), Handler)
self.threads = [threading.Thread(target=self.worker, name="http-" + str(n), daemon=True)
for n in range(config.workers)]
for thread in self.threads:
thread.start()
def get_request(self):
client, address = super().get_request()
client.settimeout(self.config.socket_timeout)
return client, address
def record(self, status):
with self.stats_lock:
self.statuses[str(status)] += 1
def process_request(self, request, address):
try:
if self.draining.is_set():
raise queue.Full()
self.jobs.put_nowait((request, address))
except queue.Full:
# Transport admission before HTTP parsing: bounded time, no extra thread.
identifier = str(uuid.uuid4())
body = json.dumps(error_body(503, "Request queue full or shutting down", identifier)).encode()
headers = ("HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
"Connection: close\r\nRetry-After: 1\r\nX-Request-Id: " + identifier +
"\r\nContent-Length: " + str(len(body)) + "\r\n\r\n").encode()
try:
request.settimeout(.05)
# Wait for the first request bytes: closing before a local client
# finishes sendall can otherwise discard the 503 with a TCP reset.
try:
request.recv(16384)
except OSError:
pass
request.sendall(headers + body)
request.shutdown(socket.SHUT_WR)
request.settimeout(.01)
for _ in range(5):
if not request.recv(16384):
break
except OSError:
pass
self.shutdown_request(request)
self.record(503)
log_event(event="queue_rejected", status=503, request_id=identifier)
def worker(self):
while not self.draining.is_set():
try:
request, address = self.jobs.get(timeout=.1)
except queue.Empty:
continue
with self.stats_lock:
self.active += 1
self.clients.add(request)
try:
self.finish_request(request, address)
except OSError:
pass
except Exception:
log_event(event="worker_error")
finally:
self.shutdown_request(request)
with self.stats_lock:
self.active -= 1
self.clients.discard(request)
self.jobs.task_done()
def server_close(self):
self.draining.set()
with self.stats_lock:
clients = list(self.clients)
for client in clients:
try:
client.shutdown(socket.SHUT_RDWR)
except OSError:
pass
while True:
try:
request, _ = self.jobs.get_nowait()
except queue.Empty:
break
self.shutdown_request(request)
self.jobs.task_done()
deadline = time.monotonic() + 5
for thread in self.threads:
thread.join(timeout=max(0, deadline - time.monotonic()))
self.store.close()
super().server_close()
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "Messenger"
sys_version = ""
def setup(self):
super().setup()
self.request_id = str(uuid.uuid4())
self.received = time.monotonic()
def log_message(self, *_):
pass
def parse_request(self):
original = self.rfile
class HeaderBudget:
remaining = 16384
def readline(self, limit=-1):
line = original.readline(min(limit, self.remaining + 1) if limit >= 0 else self.remaining + 1)
self.remaining -= len(line)
if self.remaining < 0:
raise http.client.LineTooLong("header section")
return line
self.rfile = HeaderBudget()
try:
return super().parse_request()
finally:
self.rfile = original
def send_error(self, code, message=None, explain=None):
self.reply(code if code in CODES else 400, error_body(
code if code in CODES else 400, "Invalid HTTP request", self.request_id))
def reply(self, status, body):
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
content_type = 'application/json'
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(raw)))
self.send_header("X-Request-Id", self.request_id)
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Connection", "close")
if status == 401:
challenge = ('Basic realm="messenger", charset="UTF-8"'
if urlsplit(self.path).path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
self.send_header('WWW-Authenticate', challenge)
if status in (429, 503):
self.send_header("Retry-After", "1")
self.end_headers()
self.close_connection = True
self.wfile.write(raw)
self.server.record(status)
# No URL query, header, username, text, or exception detail in logs.
log_event(event="request", method=self.command, status=status,
request_id=self.request_id, elapsed_ms=round(1000 * (time.monotonic() - self.received), 3))
def handle_expect_100(self):
# Do not acknowledge a large unauthorized request before checking identity.
return True
def body(self):
if self.headers.get_all("Transfer-Encoding"):
raise APIError(400, "Transfer-Encoding is unsupported")
sizes = self.headers.get_all("Content-Length", [])
if len(sizes) != 1 or not sizes[0].isascii() or not sizes[0].isdigit() or len(sizes[0]) > 10:
raise APIError(400, "One valid Content-Length is required")
size = int(sizes[0])
if size > self.server.config.max_body_bytes:
raise APIError(413, "Request body exceeds byte limit")
if self.headers.get_content_type() != "application/json":
raise APIError(415, "Expected application/json")
if self.headers.get("Expect", "").lower() == "100-continue":
self.send_response_only(100)
self.end_headers()
raw = self.rfile.read(size)
if len(raw) != size:
raise APIError(400, "Incomplete request body")
def pairs(values):
result = {}
for key, value in values:
if key in result:
raise ValueError("duplicate key")
result[key] = value
return result
def constant(_):
raise ValueError("non JSON constant")
try:
return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=constant)
except (ValueError, UnicodeError, RecursionError):
raise APIError(400, "Invalid JSON") from None
def dispatch(self):
try:
url = urlsplit(self.path)
except ValueError:
raise APIError(400, "Invalid request target") from None
path, method = url.path, self.command
store = self.server.store
if self.server.draining.is_set():
raise APIError(503, "Shutting down")
if method == 'GET' and path == '/health/live':
return 200, {'status': 'ok'}
if method == 'GET' and path == '/health/ready':
store.ready()
return 200, {'status': 'ok'}
if path == '/api/v1/users' and method == 'POST':
return 201, store.create_user(self.body())
headers = self.headers.get_all('Authorization', [])
authorization = headers[0] if len(headers) == 1 else None
if path == '/api/v1/auth/sessions' and method == 'POST':
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
raise APIError(400, 'Login takes no request body')
return 201, store.login(authorization)
actor, session_key = store.auth(authorization)
if method == 'DELETE' and path == '/api/v1/auth/sessions/current':
store.cache.delete(session_key)
return 204, None
if method == 'GET' and path == '/api/v1/users/me':
return 200, store.me(actor)
parts = path.strip("/").split("/")
chat_id = None
if len(parts) >= 4 and parts[:3] == ["api", "v1", "chats"]:
chat_id = store.chat_access(actor, parts[3])
if method == "POST" and path == "/api/v1/chats":
return 201, store.create_chat(actor, self.body())
if method == "POST" and chat_id and len(parts) == 5 and parts[4] == "messages":
return 201, store.create_message(actor, chat_id, self.body())
if method == "GET":
if chat_id and len(parts) == 4:
return 200, store.chat(chat_id)
if path == "/api/v1/chats" or (chat_id and len(parts) == 5 and parts[4] == "messages"):
try:
query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True, max_num_fields=8)
except ValueError:
raise APIError(400, "Invalid query") from None
return 200, store.page(actor, chat_id, query)
raise APIError(404, "Object not found")
def handle_api(self):
try:
status, result = self.dispatch()
except APIError as error:
status = error.status
result = error_body(status, error.message, self.request_id)
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable, PoolTimeout, TooManyRequests, redis.RedisError):
status, result = 503, error_body(503, 'Dependency temporarily unavailable', self.request_id)
except (TimeoutError, socket.timeout):
status, result = 400, error_body(400, "Request read timeout", self.request_id)
except Exception:
status, result = 500, error_body(500, "Internal error", self.request_id)
log_event(event="internal_error", request_id=self.request_id)
try:
self.reply(status, result)
except OSError:
pass
do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = do_OPTIONS = handle_api
def main():
logging.basicConfig(level=logging.INFO, format="%(message)s")
# Pool diagnostics may include connection details; application emits sanitized errors.
logging.getLogger('psycopg.pool').disabled = True
config = Config.from_env()
server = Server(config)
def stop(signum, _):
if not server.draining.is_set():
server.draining.set()
log_event(event="shutdown_started", signal=signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
log_event(event="started", port=config.app_port, workers=config.workers)
try:
server.serve_forever(poll_interval=.1)
finally:
server.server_close()
log_event(event="shutdown_complete")
if __name__ == "__main__":
main()
+89
View File
@@ -0,0 +1,89 @@
"""Own black-box tests; synthetic data, standard library, no tokens in output."""
import base64
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import sys
import unittest
from urllib.parse import quote
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
class API(unittest.TestCase):
def setUp(self):
self.c = Client()
def test_registration_race(self):
body = dict(username='race_' + uuid.uuid4().hex[:20], display_name='Гонка', password='Unicode:пароль🔑123')
def register(_):
return self.c.call('POST', '/api/v1/users', [201, 409], body=body)['status']
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(register, range(10)))
self.assertEqual(statuses.count(201), 1)
self.assertEqual(statuses.count(409), 9)
def test_validation_access_and_pagination(self):
self.c.call('POST', '/api/v1/users', 400,
body=dict(username='spaces_' + uuid.uuid4().hex[:20], display_name='Invalid', password=' ' * 12))
alice, bob, eve, chat = self.c.bootstrap()
path = '/api/v1/chats/' + chat['id']
for raw in (b'{', b'{"text":NaN}', b'{"text":"a","text":"b"}', b'[]'):
self.c.call('POST', path + '/messages', 400, actor=alice, raw=raw,
headers={'Content-Type': 'application/json'})
for body in ({'text': ' '}, {'text': 'hi', 'sender_id': eve['id']}, {'text': 2}, {'text': '\x00'}):
self.c.call('POST', path + '/messages', 400, actor=alice, body=body)
self.c.call('POST', path + '/messages', 404, actor=eve, body={'text': 'нет доступа'})
self.c.call('POST', '/api/v1/users', 409, body=alice['registration'])
for members in ([alice['id'], str(uuid.uuid4())], [bob['id']], [alice['id'], alice['id']]):
self.c.call('POST', '/api/v1/chats', 400, actor=alice,
body={'title': 'Invalid membership', 'member_ids': members})
sent = [self.c.message(alice, chat, 'Страница ' + str(i))['json'] for i in range(7)]
read, cursor = [], None
while True:
query = '?limit=2' + ('&cursor=' + quote(cursor, safe='') if cursor else '')
page = self.c.call('GET', path + '/messages' + query, actor=bob, schema='MessagePage')['json']
read.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
break
self.assertEqual(read, sorted(sent, key=lambda x: (x['created_at'], x['id'])))
first = self.c.call('GET', path + '/messages?limit=1', actor=alice, schema='MessagePage')['json']
other = self.c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json']
self.c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + first['next_cursor'], 400, actor=alice)
self.c.call('GET', path + '/messages?cursor=broken', 400, actor=alice)
def test_concurrent_messages(self):
alice, _, _, chat = self.c.bootstrap()
def send(i):
return self.c.message(alice, chat, 'Concurrent ' + str(i))['json']['id']
with ThreadPoolExecutor(max_workers=10) as pool:
ids = list(pool.map(send, range(100)))
page = self.c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
actor=alice, schema='MessagePage')['json']
self.assertEqual(len(set(ids)), 100)
self.assertEqual({x['id'] for x in page['items']}, set(ids))
self.assertIsNone(page['next_cursor'])
def test_unicode_password_and_logout(self):
username = 'utf_' + uuid.uuid4().hex[:20]
password = '🔑:пароль' * 16 # 128 code points, far more than bcrypt's 72 bytes
self.assertEqual(len(password), 128)
self.c.call('POST', '/api/v1/users', 201,
body=dict(username=username, display_name='Unicode', password=password), schema='User')
def login(value, expected):
auth = base64.b64encode((username + ':' + value).encode()).decode()
return self.c.call('POST', '/api/v1/auth/sessions', expected,
headers={'Authorization': 'Basic ' + auth}, schema='Session' if expected == 201 else None)
actor = {'token': login(password, 201)['json']['token']}
login('incorrect-password', 401)
self.c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
self.c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
self.c.call('GET', '/api/v1/users/me', 401, actor=actor)
if __name__ == '__main__':
unittest.main(verbosity=2)
+59
View File
@@ -0,0 +1,59 @@
"""Run inside API container: directly observe rollback and stored password KDF."""
from pathlib import Path
import sys
import subprocess
import unittest
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
from messenger.core import APIError, Config, Store, password_matches
class Database(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.store = Store(Config.from_env())
@classmethod
def tearDownClass(cls):
cls.store.close()
def test_rollback_after_chat_insert(self):
actor = self.store.create_user(dict(username='rollback_' + uuid.uuid4().hex[:16],
display_name='Rollback', password='rollback-password'))['id']
with self.store.pool.connection() as conn:
before = conn.execute('SELECT count(*) AS n FROM chats').fetchone()['n']
members_before = conn.execute('SELECT count(*) AS n FROM chat_members').fetchone()['n']
# First membership succeeds, second violates FK after the chat was inserted.
with self.assertRaises(APIError) as raised:
self.store.create_chat(actor, dict(title='Must rollback', member_ids=[actor, str(uuid.uuid4())]))
self.assertEqual(raised.exception.status, 400)
with self.store.pool.connection() as conn:
self.assertEqual(conn.execute('SELECT count(*) AS n FROM chats').fetchone()['n'], before)
self.assertEqual(conn.execute('SELECT count(*) AS n FROM chat_members').fetchone()['n'], members_before)
def test_password_salted_kdf(self):
password = 'identical-password'
ids = [self.store.create_user(dict(username='salt_' + uuid.uuid4().hex[:20],
display_name='Salt', password=password))['id'] for _ in range(2)]
with self.store.pool.connection() as conn:
hashes = [conn.execute('SELECT password_hash FROM users WHERE id=%s', (i,)).fetchone()['password_hash'] for i in ids]
self.assertNotEqual(*hashes)
for value in hashes:
self.assertTrue(value.startswith('scrypt$16384$8$1$'))
self.assertTrue(password_matches(password, value))
self.assertNotEqual(value, password)
def test_repeat_migrations(self):
def counts():
with self.store.pool.connection() as conn:
return [conn.execute('SELECT count(*) AS n FROM ' + table).fetchone()['n']
for table in ('users', 'chats', 'chat_members', 'messages', 'schema_migrations')]
before = counts()
for _ in range(2):
subprocess.run([sys.executable, '-m', 'messenger.migrate'], check=True)
self.assertEqual(counts(), before)
if __name__ == '__main__':
unittest.main(verbosity=2)
+14
View File
@@ -0,0 +1,14 @@
"""Use a separate API process with SESSION_TTL_SECONDS=2, sharing real stores."""
from pathlib import Path
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check, timestamp
c = Client()
actor = c.user('expiry')
response = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
time.sleep(2.2)
c.call('GET', '/api/v1/users/me', 401, actor=actor)
print('PASS: session accepted before expiry; rejected after 2.2s with SESSION_TTL_SECONDS=2.')
+50
View File
@@ -0,0 +1,50 @@
"""Host-side operator test: stop one lab dependency, assert bounded failures, restore."""
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
c = Client(timeout=4)
actor, _, _, chat = c.bootstrap()
original = c.message(actor, chat, 'Before dependency outage')['json']
path = '/api/v1/chats/' + chat['id'] + '/messages'
summary = []
for service in ('redis', 'db'):
subprocess.run(['docker', 'compose', 'stop', service], cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
try:
timings = []
for _ in range(3):
response = c.call('GET', '/api/v1/users/me', 503, actor=actor)
check(response['json']['error']['code'] == 'unavailable', 'wrong dependency error')
check(response['elapsed'] < 3.5, 'failure exceeded response budget')
timings.append(response['elapsed'])
c.call('POST', path, 503, actor=actor, body={'text': 'Must not commit while unavailable'})
c.call('GET', '/health/ready', 503)
c.call('GET', '/health/live', schema='Health')
c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Bearer ' + 'A' * 43}) if service == 'db' else c.call(
'GET', '/api/v1/users/me', 503, headers={'Authorization': 'Bearer ' + 'A' * 43})
finally:
started = time.monotonic()
subprocess.run(['docker', 'compose', 'start', service], cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 30
while True:
try:
c.call('GET', '/health/ready', schema='Health')
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
break
except Failure:
check(time.monotonic() < deadline, 'dependency did not recover in 30 seconds')
time.sleep(.25)
recovery = time.monotonic() - started
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
check(page['items'] == [original], 'lost confirmation or wrote during dependency outage')
summary.append(dict(service=service, errors=3, status=503, seconds=timings,
recovery_seconds=round(recovery, 3), original_session_preserved=True,
original_message_preserved=True, anonymous_status=401))
print(json.dumps(summary, indent=2))