From 1da034a8413bc3802a77661719bb8b4992c8f061 Mon Sep 17 00:00:00 2001 From: Xabenskie Date: Mon, 2 Nov 2026 10:34:35 +0300 Subject: [PATCH 1/2] test commit --- tests/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/client.py b/tests/client.py index f0499a29..bf469357 100644 --- a/tests/client.py +++ b/tests/client.py @@ -11,7 +11,7 @@ import time import urllib.error import urllib.request import uuid - +# ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) -- 2.54.0 From 990a36c95f18b959a7c9f43d74999e9b8eb466a8 Mon Sep 17 00:00:00 2001 From: Xabenskie Date: Mon, 2 Nov 2026 11:02:27 +0300 Subject: [PATCH 2/2] [+] lab 2 --- .dockerignore | 3 + .env.example | 17 +- .github/workflows/application.yml | 20 +++ .gitignore | 2 + Dockerfile | 15 ++ Makefile | 24 +++ REPORT.md | 117 ++++++++++-- compose.yaml | 55 ++++++ evidence/check.txt | 1 + evidence/dependency-failures.json | 22 +++ evidence/environment.json | 35 ++++ evidence/explain-history.json | 151 ++++++++++++++++ evidence/fresh-migrations.txt | 6 + evidence/load-output.txt | 17 ++ evidence/load.json | 17 ++ evidence/openapi.txt | 1 + evidence/persistence.json | 52 ++++++ evidence/persistence.txt | 3 + evidence/resources.json | 288 ++++++++++++++++++++++++++++++ evidence/smoke.txt | 1 + evidence/student-tests.txt | 2 + migrations/001_initial.sql | 34 ++++ requirements.lock | 45 +++++ requirements.txt | 10 ++ scripts/bootstrap.sh | 28 +++ scripts/capture_load.py | 56 ++++++ scripts/check_fresh_migrations.sh | 16 ++ scripts/explain_history.py | 46 +++++ scripts/restart.sh | 5 + scripts/verify_grade4.py | 82 +++++++++ scripts/verify_persistence.py | 40 +++++ src/messenger/__init__.py | 0 src/messenger/app.py | 249 ++++++++++++++++++++++++++ src/messenger/db.py | 42 +++++ src/messenger/errors.py | 19 ++ src/messenger/migrate.py | 29 +++ src/messenger/models.py | 60 +++++++ src/messenger/pagination.py | 48 +++++ src/messenger/security.py | 49 +++++ student-tests/test_api.py | 141 +++++++++++++++ tests/client.py | 2 +- 41 files changed, 1826 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/check.txt create mode 100644 evidence/dependency-failures.json create mode 100644 evidence/environment.json create mode 100644 evidence/explain-history.json create mode 100644 evidence/fresh-migrations.txt create mode 100644 evidence/load-output.txt create mode 100644 evidence/load.json create mode 100644 evidence/openapi.txt create mode 100644 evidence/persistence.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/resources.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/student-tests.txt create mode 100644 migrations/001_initial.sql create mode 100644 requirements.lock create mode 100644 requirements.txt create mode 100755 scripts/bootstrap.sh create mode 100644 scripts/capture_load.py create mode 100755 scripts/check_fresh_migrations.sh create mode 100644 scripts/explain_history.py create mode 100755 scripts/restart.sh create mode 100644 scripts/verify_grade4.py create mode 100644 scripts/verify_persistence.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/app.py create mode 100644 src/messenger/db.py create mode 100644 src/messenger/errors.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/models.py create mode 100644 src/messenger/pagination.py create mode 100644 src/messenger/security.py create mode 100644 student-tests/test_api.py diff --git a/.dockerignore b/.dockerignore index bc16e556..9c4c0b6c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.superpowers +.pytest_cache diff --git a/.env.example b/.env.example index 714c8076..feef8fb6 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,8 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Created locally by scripts/bootstrap.sh; do not commit .env or secrets/. +# Compose passes SESSION_TTL_SECONDS into API. Other fields are documented defaults. SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# In container: DATABASE_URL_FILE=/run/secrets/database_url +# In container: CACHE_URL_FILE=/run/secrets/cache_url +# API binds 0.0.0.0:8080 inside Docker; host port is 127.0.0.1:8080. +# PostgreSQL stores sessions; Redis caches only immutable User profiles (TTL 60s). +# POOL_MAX_SIZE=8, DB connect timeout=3s, pool wait=2s, SQL timeout=3s. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..3132a877 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,20 @@ +name: Application acceptance +on: [push, pull_request] +permissions: + contents: read +jobs: + api: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: make up + - run: make check test test-student + - run: make verify-persistence + - run: python3 scripts/verify_grade4.py + - run: make explain > evidence/explain-history.json + - if: always() + run: docker compose down diff --git a/.gitignore b/.gitignore index b5a71896..506449e0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +.pytest_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..399a3b78 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.13.5-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt requirements-dev.txt requirements.lock ./ +RUN pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home messenger +COPY src/ src/ +COPY migrations/ migrations/ +COPY student-tests/ student-tests/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +USER messenger +EXPOSE 8080 +CMD ["sh", "-c", "python -m messenger.migrate && exec uvicorn messenger.app:create_app --factory --host 0.0.0.0 --port 8080 --no-access-log"] diff --git a/Makefile b/Makefile index 9ecb223e..ec7a00bf 100644 --- a/Makefile +++ b/Makefile @@ -24,3 +24,27 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: bootstrap up down test-student test-grade4 explain verify-persistence +bootstrap: + ./scripts/bootstrap.sh +up: bootstrap + docker compose up -d --build --wait --wait-timeout 120 +down: + docker compose down +test-student: + docker compose exec -T api python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache +test-grade4: test-student + $(PYTHON) scripts/verify_grade4.py +explain: + docker compose exec -T api python scripts/explain_history.py +verify-persistence: + $(PYTHON) scripts/verify_persistence.py + +.PHONY: load-evidence +load-evidence: + $(PYTHON) scripts/capture_load.py + +.PHONY: test-fresh +test-fresh: + ./scripts/check_fresh_migrations.sh diff --git a/REPORT.md b/REPORT.md index e42f2b05..ded31e2d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,128 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Лукьянов К.Р. БПИ2303 +- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не заявлен) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, pytest; документация PostgreSQL (transactions, EXPLAIN), Redis (cache-aside, persistence), Psycopg (connection pools), Docker (Dockerfile reference, Compose). Реализация выполнена совместно мной и Codex: часть кода, тестов и отчёта написана с помощью Codex, результат проверен запуском тестов, описанных ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker с Compose v2 (проверено на Docker 29.8.2, macOS), Python ≥ 3.9 на хосте для публичных тестов и скриптов (стандартная библиотека), Make. PostgreSQL и Redis поднимаются из этого репозитория; устанавливать их на хосте не нужно. + +```sh +git clone && cd <каталог-fork-лабораторной-2> +make up # создаёт локальные secrets/.env, собирает образ, поднимает стенд и ждёт ready + +# Эквивалент make up: +# ./scripts/bootstrap.sh +# docker compose up -d --build --wait --wait-timeout 120 + +curl -fsS http://localhost:8080/health/ready +docker compose ps # api, postgres, redis — healthy + +make check # структура skeleton, ссылки, синтаксис публичных Python-проверок +docker compose exec -T api python scripts/validate_openapi.py # полная OpenAPI-валидация +make test # публичный smoke +make test-student # собственные интеграционные тесты (pytest, 7 тестов) +make test-fresh # миграции и 7 тестов на отдельных пустых volumes +make verify-persistence # публичный restart-тест + проверка реального перезапуска контейнеров +make test-grade4 # собственные тесты + отказ Redis/БД и восстановление +make explain # EXPLAIN (ANALYZE, BUFFERS) на 10 000 сообщениях +make load-evidence # 30 секунд нагрузки + CPU/RAM и параметры окружения +``` + +API опубликован только на `127.0.0.1:8080`. `make up` ожидает готовности сервисов; миграции выполняются перед запуском Uvicorn. Повторная сборка использует слой зависимостей, пока не меняются файлы requirements. + +`bootstrap.sh` один раз создаёт случайный пароль БД, файлы `secrets/postgres_password`, `secrets/database_url`, `secrets/cache_url` и локальный `.env`. Повторный запуск не меняет существующие значения. Каталог secrets имеет права 0700 на хосте; файлы внутри доступны uid 10001 контейнера через Compose secrets. `.env` и secrets исключены из Git и Docker build context. При сохранённых volumes нужно сохранять и secrets: изменение файла пароля не меняет пароль уже инициализированной PostgreSQL. + +Безопасная остановка: `make down` (`docker compose down`) — именованные volumes сохраняются, после `make up` доступны прежние данные и ещё действующие сессии. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Скрипт `scripts/restart.sh` эту команду не использует: он перезапускает API, PostgreSQL и Redis с сохранением volumes. + +`make test-fresh` создаёт отдельный временный Compose-проект без публикации портов и после проверки удаляет только его временные volumes. Проверки отказов не запускать одновременно со smoke/load: они намеренно останавливают зависимости. Синтетические данные обычных тестов остаются в основной БД, так как обязательный API не содержит delete. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +клиент ──HTTP──► uvicorn (PID 1, 1 worker) ──► FastAPI ──► PostgreSQL + │ данные и сессии + └────────► Redis + кэш профиля, TTL 60 с +``` + +- **Источник истины** — PostgreSQL: пользователи, чаты, членство, сообщения и сессии. Базовые endpoint пользователей/чатов/сообщений реализованы в этой работе; сохранён обязательный минимум lab1, идентификация через X-User-Id заменена Basic → Bearer согласно переходу lab1 → lab2. +- **Схема** (`migrations/001_initial.sql`): `users` — UUID PK и UNIQUE username; `chats` — UUID PK; `chat_members` — PK(chat_id,user_id), FK на чат и пользователя; `messages` — UUID PK, FK chat_id и составной FK(chat_id,sender_id) на членство; `sessions` — token_hash PK, FK user_id, expires_at. Есть ограничения длины/формата полей и индексы для членства, истории и срока сессий. +- **Транзакционные границы / гонки.** Регистрация защищена UNIQUE в БД: проверка не опирается на предварительный SELECT. Чат и все участники создаются одной транзакцией. Неизвестный участник вызывает FK-ошибку после INSERT чата, откатывается вся операция. Проверка членства и вставка сообщения — в одной транзакции. Успешный ответ отправляется после COMMIT. SQL-параметры передаются отдельно от текста запроса. +- **Миграции** (`src/messenger/migrate.py`): SQL-файлы версионируются через `schema_migrations`; применённый checksum проверяется. Advisory lock сериализует параллельные запуски. Изменение схемы и запись версии атомарны. Повторный запуск не дублирует данные. Ручная команда: `docker compose exec -T api python -m messenger.migrate`. +- **Пароли и вход.** Argon2id с индивидуальной солью: memory_cost 19456 KiB, time_cost 2, parallelism 1. Для неизвестного username выполняется проверка dummy hash. Basic применяется только на POST `/api/v1/auth/sessions`, без JSON body; пароль UTF-8, двоеточие поддерживается. Неверные credentials → одинаковый 401 с `WWW-Authenticate: Basic realm="messenger", charset="UTF-8"`. +- **Сессии.** `secrets.token_urlsafe(32)` создаёт opaque Bearer с энтропией 256 бит. В БД сохраняются SHA-256 токена, user_id и абсолютный expires_at; исходный токен возвращается только клиенту. Каждый защищённый запрос проверяет сессию в PostgreSQL. Logout удаляет её; истёкший/отозванный/неизвестный токен → 401 с Bearer challenge. Рестарт не продлевает TTL. Basic и X-User-Id не дают доступ к бизнес-endpoint. +- **Redis** — cache-aside для GET `/api/v1/users/me` после проверки сессии в БД. Ключ `user:` хранит только публичный JSON User на 60 с; miss читает профиль из PostgreSQL, hit экономит отдельный SELECT профиля. Пароли, хеши паролей и токены в кэш не попадают. Изменения/удаления профиля в обязательном API нет, поэтому инвалидация по событию не нужна; время жизни ограничено TTL. Потеря/eviction кэша допустимы. При недоступности Redis выполняется чтение из БД; кэш никогда не подтверждает аутентификацию. Redis использует AOF и volume, но сохранность сессий обеспечивается PostgreSQL. +- **Время и порядок.** UUID назначает сервер, `created_at` — PostgreSQL через `clock_timestamp()`, сериализация UTC с Z. История упорядочена по `(created_at,id)`, часы не используются как уникальный идентификатор. +- **Пагинация** — keyset cursor: base64url(JSON `[scope,created_at,id]`), область `messages:` или `chats:`. Cursor другого ресурса → 400. Предикат строго `>`, выборка limit+1 определяет next_cursor без OFFSET. Индекс `messages_history_idx(chat_id,created_at,id)` совпадает с фильтром и порядком. История не кэшируется: следующему чтению видна подтверждённая запись из БД. +- **Доступ и ошибки.** Действующая сессия определяет пользователя, затем проверяется членство; чужой и несуществующий чат → 404. Единый формат `{"error":{code,message,request_id}}`; request_id совпадает с X-Request-Id во всех ответах, включая health и ошибки. Malformed JSON, неверные типы, лишние поля, пробельные строки → 400; неподдерживаемый Content-Type → 415. JSON разбирается как UTF-8, а не через автоматическое распознавание UTF-16/UTF-32. +- **Пулы и таймауты.** PostgreSQL: min 1/max 8 соединений, ожидание до 2 с, очередь до 16; connect_timeout 3 с, statement_timeout 3 с, lock_timeout 2 с, idle transaction timeout 5 с. Redis: до 8 соединений, connect/read timeout по 0.2 с, без автоматических retry. Ошибка БД → ограниченный 503 без SQL/credentials. Это таймауты отдельных операций, не общий SLA запроса при любой нагрузке. +- **Готовность и логи.** `/health/live` проверяет процесс; `/health/ready` выполняет SQL. PostgreSQL критичен, Redis необязателен для синхронного пути. Access-лог Uvicorn отключён; при непредвиденной ошибке пишутся request_id и класс исключения, без тела сообщения, токена, SQL и stack trace. + +Конфигурация и роли сервисов: + +| Переменная / service | Значение | Использование | +| --- | --- | --- | +| `SESSION_TTL_SECONDS` (.env) | `3600` по умолчанию | абсолютный срок сессии; в тесте expiry — 2 с | +| `DATABASE_URL_FILE` | `/run/secrets/database_url` | адрес и credentials PostgreSQL | +| `CACHE_URL_FILE` | `/run/secrets/cache_url` | адрес Redis | +| `POOL_MAX_SIZE` | `8` в Compose | максимальный размер пула PostgreSQL | +| `api` | локальный `mtusi-lab2-api:lab2`; 1 CPU / 512 MiB | bind 0.0.0.0:8080 внутри контейнера, на хосте 127.0.0.1:8080; SQL healthcheck; без volume | +| `postgres` | `postgres:17.5-bookworm`; 1 CPU / 512 MiB | только сеть Compose, `POSTGRES_PASSWORD_FILE`; volume `postgres_data`; `pg_isready` | +| `redis` | `redis:7.4.4-bookworm`; 0.5 CPU / 128 MiB | только сеть Compose; volume `redis_data`; `redis-cli ping`; maxmemory 64 MiB/allkeys-lru | + +**Dockerfile:** базовый образ `python:3.13.5-slim`, зависимости включая транзитивные закреплены `==` в `requirements.lock`. Основные библиотеки — в `requirements.txt`, валидатор — в `requirements-dev.txt`. Сначала копируются зависимости и выполняется pip install, затем код, миграции и проверки. Пользователь `messenger` (uid 10001); код принадлежит root. CMD сначала запускает мигратор, затем `exec uvicorn`, поэтому сервер становится PID 1. Адрес/порт заданы командой Dockerfile; readiness healthcheck — в Compose. `.dockerignore` исключает Git, `.env`, secrets, сертификаты, volumes, кэши и evidence. В образ включены публичные и собственные проверки для воспроизводимого запуска без локальных Python-зависимостей. ## Доказательства критериев +Все прогоны 2026-11-02, macOS, Apple M4 Pro / 24 GiB RAM, Docker Desktop 29.8.2. Docker VM: 12 CPU, около 7.75 GiB RAM. Образ `mtusi-lab2-api:lab2` из текущей рабочей копии, ветка `solution/bpi2303-lukyanov`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke (3) | `make test` | `PASS: lab2 public API scenario` | `evidence/smoke.txt` | +| Контракт OpenAPI | `make check`; валидатор в контейнере | оба PASS | `evidence/check.txt`, `evidence/openapi.txt` | +| Базовый API, членство, история, переход на Bearer (3) | smoke + `make test-student` | 7/7 PASS; Unicode, пагинация, чужой чат, неверные типы/поля/cursor | `evidence/smoke.txt`, `evidence/student-tests.txt` | +| PostgreSQL, PK/FK/UNIQUE, атомарное создание чата (3) | схема + `test_chat_rollback_after_insert` | неизвестный участник → 400; вставленный чат и членства откатились | `migrations/001_initial.sql`, `evidence/student-tests.txt` | +| Пароли через KDF, срок сессии и отзыв (3) | smoke + собственные тесты auth/expiry/storage | Argon2id в БД, hash вместо исходного токена; logout → 204, повтор → 401; Basic/X-User-Id не дают бизнес-доступ | `evidence/smoke.txt`, `evidence/student-tests.txt` | +| Данные и действующие сессии переживают restart всех сервисов (3) | `make verify-persistence` | прежний токен читает прежние данные; StartedAt API/БД/Redis изменился, volumes те же | `evidence/persistence.txt`, `evidence/persistence.json` | +| Миграции с нуля и повторный запуск без дублей (3) | `make test-fresh`; `test_session_storage_and_migrations` | отдельные пустые volumes; два запуска мигратора и 7/7 PASS; повторение не меняет число строк | `evidence/fresh-migrations.txt`, `evidence/student-tests.txt` | +| Единый локальный запуск, именованные volumes, secrets вне Git (3) | `make up`; Compose, bootstrap и ignore-файлы | три healthy-сервиса; secrets генерируются локально | `compose.yaml`, `scripts/bootstrap.sh`, `evidence/persistence.json` | +| Redis с TTL, источник истины и политика инвалидации (4) | `scripts/verify_grade4.py` | miss заполняет кэш, TTL 60 с; повторный запрос увеличил keyspace_hits | `evidence/dependency-failures.json` | +| Параметризованный SQL, ограниченные пулы и таймауты (4) | `src/messenger/db.py`, `app.py`; отказ БД | ошибку зависимости получает клиент, внутренние данные не раскрываются | исходники, `evidence/dependency-failures.json` | +| Гонка одинакового username и rollback (4) | `test_registration_race`, `test_chat_rollback_after_insert` | из 6 регистраций один 201 и пять 409; составная операция полностью откатилась | `evidence/student-tests.txt` | +| Индекс истории на ≥ 10 000 сообщений (4) | `make explain` | Index Scan по messages_history_idx: первая страница 0.011 мс, глубокая 0.014 мс | `evidence/explain-history.json` | +| Короткий SESSION_TTL_SECONDS, logout, неверный пароль (4) | smoke + `test_session_expiry`, `test_auth_unicode_logout_and_cache` | TTL=2: до срока 200, после 401; отозванный токен не проходит при тёплом кэше | `evidence/smoke.txt`, `evidence/student-tests.txt` | +| Недоступность Redis/БД без ослабления авторизации (4) | `make test-grade4` | Redis: fallback 200; PostgreSQL: 503; без Bearer всегда 401; сервисы восстановлены | `evidence/dependency-failures.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Повторное сохранение вывода: `make test-student > evidence/student-tests.txt`, `python3 scripts/verify_grade4.py > evidence/dependency-failures.json`, `docker compose exec -T api python scripts/explain_history.py > evidence/explain-history.json`. `make verify-persistence` сам обновляет JSON состояний контейнеров, `make load-evidence` — нагрузочные результаты и параметры окружения. Пароли и токены в evidence не сохраняются. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Исследовательский трек 5 не заявлен. Проверки уровня 4 выполняются на реальных PostgreSQL и Redis; сценарий отказов создаёт синтетического пользователя/сессию, прогревает профиль и по очереди останавливает зависимости через Compose. Каждый сервис возвращается в `finally`, после запуска БД ожидается SQL-readiness. По одному прогону каждого отказа. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| 6 конкурентных регистраций одного username | ровно один пользователь, остальные запросы конфликтуют | один 201, пять 409 | — / дублей нет | +| Неизвестный участник после INSERT чата | откат всей транзакции | 400, чат и членства не сохранились | — / частичного состояния нет | +| SESSION_TTL_SECONDS=2 | токен перестаёт работать по истечении срока | `/users/me`: сначала 200, через 2.1 с — 401 | — / истечение ожидаемо | +| Redis остановлен | профиль читается из БД, проверка сессии сохраняется | 200 за 0.0165 с, ready200, аноним401; logout204, отозванный токен401 и после восстановления Redis | данные в PostgreSQL сохранены | +| PostgreSQL остановлен | live200, ready503, защищённый запрос503; без Bearer401 | 503 за 2.0138 с; после старта БД прежняя сессия снова работает | 0.6373 с от возврата команды start до ready и чтения / потери не обнаружены | +| API + PostgreSQL + Redis restart с volumes | прежний токен читает подтверждённые данные без повторного входа | public persistence PASS; реальные времена запуска изменились, volumes прежние | потери подтверждённого сообщения и сессии не обнаружены | + +**План запроса:** `scripts/explain_history.py` создаёт отдельный чат с 10 000 сообщениями, выполняет ANALYZE и EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON). Проверяются первая страница и страница после 9001-й записи, обе с LIMIT 51. На обеих используется составной индекс, без полного обхода истории на каждую страницу. OFFSET применяется только для выбора экспериментальной точки, в запросах приложения его нет; настройки планировщика не отключаются. Полные планы и buffer-счётчики сохранены в `evidence/explain-history.json`. + +**Нагрузочный прогон:** `tests/load.py`, closed-loop POST сообщений, 30 с, concurrency 4, без автоматических retry. Перед запуском стенд уже прогрет функциональными тестами, отдельного измеряемого прогрева нагрузки нет. Один сохранённый прогон: 19 735 успешных запросов, 0 ошибок (0%), 655.021 successful RPS; p50/p95/p99 = 5.074/11.752/26.651 мс. До запуска в БД было 29 313 сообщений, после — 49 048. CPU/RAM контейнеров во время нагрузки — `evidence/resources.json`, окружение/лимиты/размер данных — `evidence/environment.json`, исходный summary — `evidence/load.json`. + +Эти числа характеризуют локальный прогон, не максимальную пропускную способность. Closed-loop имеет coordinated omission, latency считается только для успешных ответов; клиент и Docker работают на одном ноутбуке. Время EXPLAIN — время SQL на прогретых данных, не полная задержка HTTP-запроса. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host и одна PostgreSQL: потеря машины/диска не покрывается. Restart с volume не заменяет backup и не обеспечивает HA. Стандартные fsync/synchronous_commit не отключались, но отказ питания/диска не моделировался. +- Между страницами нет общего snapshot: конкурентная запись с ключом до cursor может остаться позади обхода. Порядок COMMIT не обязан совпадать с `(created_at,id)`; перестановка системных часов тоже влияет на порядок вставок. +- Каждый защищённый запрос проверяет сессию в PostgreSQL, даже при тёплом Redis. Кэш экономит только отдельный SELECT профиля; это осознанный компромисс для простого корректного отзыва. +- Истёкшие сессии отвергаются по expires_at, но фоновая очистка старых строк не добавлена. При появлении update/delete профиля понадобится инвалидация кэша. +- HTTP/Basic используется только локально с синтетическими данными. TLS, rate limiting, разделение привилегий мигратора/runtime и управление секретами остаются дальнейшим работам. +- Нет UI, WebSocket, редактирования/удаления сообщений и изменения участников — обязательный API этого не требует. +- JSON разбирается до выполнения зависимостей: malformed JSON без Bearer может дать 400, а не 401. Для корректных тел проверка сессии предшествует проверке доступа к объекту. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..3843bee1 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,55 @@ +name: mtusi-lab2 +services: + postgres: + image: postgres:17.5-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + redis: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + volumes: [redis_data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 0.5 + api: + image: mtusi-lab2-api:lab2 + build: . + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + POOL_MAX_SIZE: '8' + secrets: [database_url, cache_url] + ports: ['127.0.0.1:8080:8080'] + depends_on: + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2)"] + interval: 2s + timeout: 3s + retries: 30 + mem_limit: 512m + cpus: 1.0 +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} +volumes: + postgres_data: + redis_data: diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/dependency-failures.json b/evidence/dependency-failures.json new file mode 100644 index 00000000..48078a52 --- /dev/null +++ b/evidence/dependency-failures.json @@ -0,0 +1,22 @@ +{ + "cache_ttl_seconds": 60, + "cache_miss_hit": "PASS", + "redis_outage": { + "result": "PASS", + "fallback_seconds": 0.0165, + "readiness": 200, + "anonymous": 401, + "logout": 204, + "revoked": 401 + }, + "postgres_outage": { + "result": "PASS", + "authenticated_status": 503, + "response_seconds": 2.0138, + "readiness": 503, + "liveness": 200, + "anonymous": 401 + }, + "postgres_recovery_seconds": 0.6373, + "result": "PASS" +} diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..a25d02d2 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,35 @@ +{ + "captured_at_utc": "2026-11-02T07:53:02.847576+00:00", + "host_os": "macOS-27.2-arm64-arm-64bit-Mach-O", + "host_architecture": "arm64", + "docker": { + "server_version": "29.8.2", + "cpus": 12, + "memory_bytes": 8319238144, + "architecture": "aarch64" + }, + "python_in_image": "Python 3.13.5", + "dataset_before": { + "users": 48, + "chats": 14, + "messages": 29313, + "sessions": 33 + }, + "limits": { + "api": "1 CPU / 512 MiB", + "postgres": "1 CPU / 512 MiB", + "redis": "0.5 CPU / 128 MiB" + }, + "warmup": "API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.", + "repetitions": 1, + "duration_seconds": 30, + "concurrency": 4, + "host_cpu": "Apple M4 Pro", + "host_memory_bytes": 25769803776, + "dataset_after": { + "users": 51, + "chats": 15, + "messages": 49048, + "sessions": 36 + } +} diff --git a/evidence/explain-history.json b/evidence/explain-history.json new file mode 100644 index 00000000..824e8ddc --- /dev/null +++ b/evidence/explain-history.json @@ -0,0 +1,151 @@ +{ + "messages_in_chat": 10000, + "limit_plus_one": 51, + "index": "messages_history_idx (chat_id, created_at, id)", + "first_sql": "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=$1 ORDER BY created_at,id LIMIT 51", + "deep_sql": "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=$1 AND (created_at,id)>($2,$3) ORDER BY created_at,id LIMIT 51", + "first": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.29, + "Total Cost": 4.28, + "Plan Rows": 51, + "Plan Width": 78, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.007, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.29, + "Total Cost": 782.85, + "Plan Rows": 10000, + "Plan Width": 78, + "Actual Startup Time": 0.002, + "Actual Total Time": 0.005, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'f8c5bb0f-69c2-4915-a42b-0ce8e1192fb0'::uuid)", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.016, + "Triggers": [], + "Execution Time": 0.011 + }, + "deep_after_9001": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.29, + "Total Cost": 15.31, + "Plan Rows": 51, + "Plan Width": 78, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.007, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.29, + "Total Cost": 296.45, + "Plan Rows": 1005, + "Plan Width": 78, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.005, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'f8c5bb0f-69c2-4915-a42b-0ce8e1192fb0'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:09.001+00'::timestamp with time zone, 'eba8d96b-593c-4bb1-b6b6-b8f8787d228d'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.019, + "Triggers": [], + "Execution Time": 0.014 + } +} diff --git a/evidence/fresh-migrations.txt b/evidence/fresh-migrations.txt new file mode 100644 index 00000000..792cc031 --- /dev/null +++ b/evidence/fresh-migrations.txt @@ -0,0 +1,6 @@ +Applied 001_initial.sql +Migrations up to date +Migrations up to date +....... [100%] +7 passed in 2.83s +PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes. diff --git a/evidence/load-output.txt b/evidence/load-output.txt new file mode 100644 index 00000000..a1bb86a5 --- /dev/null +++ b/evidence/load-output.txt @@ -0,0 +1,17 @@ +{ + "lab": 2, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.129, + "concurrency": 4, + "successful_requests": 19735, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 655.021, + "successful_latency_ms": { + "p50": 5.074, + "p95": 11.752, + "p99": 26.651 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..a1bb86a5 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 2, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.129, + "concurrency": 4, + "successful_requests": 19735, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 655.021, + "successful_latency_ms": { + "p50": 5.074, + "p95": 11.752, + "p99": 26.651 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..ab7b9a4b --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,52 @@ +{ + "result": "PASS", + "before": { + "api": { + "container": "mtusi-lab2-api-1", + "started_at": "2026-11-02T07:51:53.895780469Z", + "image": "mtusi-lab2-api:lab2", + "volumes": [] + }, + "postgres": { + "container": "mtusi-lab2-postgres-1", + "started_at": "2026-11-02T07:46:35.78598434Z", + "image": "postgres:17.5-bookworm", + "volumes": [ + "mtusi-lab2_postgres_data" + ] + }, + "redis": { + "container": "mtusi-lab2-redis-1", + "started_at": "2026-11-02T07:46:33.463040423Z", + "image": "redis:7.4.4-bookworm", + "volumes": [ + "mtusi-lab2_redis_data" + ] + } + }, + "after": { + "api": { + "container": "mtusi-lab2-api-1", + "started_at": "2026-11-02T07:52:39.937143046Z", + "image": "mtusi-lab2-api:lab2", + "volumes": [] + }, + "postgres": { + "container": "mtusi-lab2-postgres-1", + "started_at": "2026-11-02T07:52:39.695575962Z", + "image": "postgres:17.5-bookworm", + "volumes": [ + "mtusi-lab2_postgres_data" + ] + }, + "redis": { + "container": "mtusi-lab2-redis-1", + "started_at": "2026-11-02T07:52:39.718304296Z", + "image": "redis:7.4.4-bookworm", + "volumes": [ + "mtusi-lab2_redis_data" + ] + } + }, + "public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..0861de5d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: actual API/PostgreSQL/Redis restart confirmed; volume identities unchanged. diff --git a/evidence/resources.json b/evidence/resources.json new file mode 100644 index 00000000..e72ab117 --- /dev/null +++ b/evidence/resources.json @@ -0,0 +1,288 @@ +[ + { + "at_utc": "2026-11-02T07:53:05.614124+00:00", + "containers": [ + { + "CPUPerc": "97.72%", + "MemPerc": "12.70%", + "MemUsage": "65.01MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "9" + }, + { + "CPUPerc": "12.92%", + "MemPerc": "6.64%", + "MemUsage": "34.02MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "10" + }, + { + "CPUPerc": "0.17%", + "MemPerc": "8.06%", + "MemUsage": "10.31MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:08.646084+00:00", + "containers": [ + { + "CPUPerc": "99.03%", + "MemPerc": "12.79%", + "MemUsage": "65.5MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "13.55%", + "MemPerc": "7.19%", + "MemUsage": "36.81MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.15%", + "MemPerc": "8.46%", + "MemUsage": "10.83MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:11.679968+00:00", + "containers": [ + { + "CPUPerc": "99.08%", + "MemPerc": "12.66%", + "MemUsage": "64.81MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "13.57%", + "MemPerc": "7.30%", + "MemUsage": "37.39MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.14%", + "MemPerc": "8.09%", + "MemUsage": "10.36MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:14.692861+00:00", + "containers": [ + { + "CPUPerc": "97.61%", + "MemPerc": "12.89%", + "MemUsage": "65.98MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "15.22%", + "MemPerc": "7.38%", + "MemUsage": "37.76MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.98%", + "MemPerc": "8.07%", + "MemUsage": "10.33MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:17.726059+00:00", + "containers": [ + { + "CPUPerc": "96.77%", + "MemPerc": "12.73%", + "MemUsage": "65.17MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "11.06%", + "MemPerc": "7.48%", + "MemUsage": "38.31MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.15%", + "MemPerc": "8.26%", + "MemUsage": "10.57MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:19.753444+00:00", + "containers": [ + { + "CPUPerc": "97.91%", + "MemPerc": "12.80%", + "MemUsage": "65.55MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "11.11%", + "MemPerc": "7.62%", + "MemUsage": "39.01MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.14%", + "MemPerc": "8.44%", + "MemUsage": "10.81MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:22.788465+00:00", + "containers": [ + { + "CPUPerc": "99.47%", + "MemPerc": "14.61%", + "MemUsage": "74.81MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "11" + }, + { + "CPUPerc": "13.46%", + "MemPerc": "7.69%", + "MemUsage": "39.36MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.14%", + "MemPerc": "8.44%", + "MemUsage": "10.8MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:25.815258+00:00", + "containers": [ + { + "CPUPerc": "97.24%", + "MemPerc": "12.82%", + "MemUsage": "65.66MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "12.32%", + "MemPerc": "7.77%", + "MemUsage": "39.76MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.97%", + "MemPerc": "8.43%", + "MemUsage": "10.79MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:28.841018+00:00", + "containers": [ + { + "CPUPerc": "97.70%", + "MemPerc": "12.78%", + "MemUsage": "65.42MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "15.64%", + "MemPerc": "7.88%", + "MemUsage": "40.32MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.96%", + "MemPerc": "8.26%", + "MemUsage": "10.57MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:31.872760+00:00", + "containers": [ + { + "CPUPerc": "97.94%", + "MemPerc": "14.49%", + "MemUsage": "74.17MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "11" + }, + { + "CPUPerc": "11.67%", + "MemPerc": "8.13%", + "MemUsage": "41.64MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "0.15%", + "MemPerc": "8.04%", + "MemUsage": "10.29MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T07:53:34.904971+00:00", + "containers": [ + { + "CPUPerc": "35.44%", + "MemPerc": "12.81%", + "MemUsage": "65.61MiB / 512MiB", + "Name": "mtusi-lab2-api-1", + "PIDs": "10" + }, + { + "CPUPerc": "9.82%", + "MemPerc": "8.09%", + "MemUsage": "41.44MiB / 512MiB", + "Name": "mtusi-lab2-postgres-1", + "PIDs": "11" + }, + { + "CPUPerc": "1.84%", + "MemPerc": "8.22%", + "MemUsage": "10.53MiB / 128MiB", + "Name": "mtusi-lab2-redis-1", + "PIDs": "6" + } + ] + } +] diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..92154196 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..986f3d7f --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,2 @@ +....... [100%] +7 passed in 2.83s diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..7b8c5338 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_history_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..55da8a0c --- /dev/null +++ b/requirements.lock @@ -0,0 +1,45 @@ +annotated-types==0.8.0 +anyio==4.9.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +attrs==26.1.0 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +iniconfig==2.3.1 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +packaging==26.3 +pathable==0.4.4 +pluggy==1.7.0 +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +pycparser==3.12 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.21.0 +pytest==8.4.0 +PyYAML==6.0.3 +redis==6.2.0 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +sniffio==1.3.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c0c5c8b7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +argon2-cffi==25.1.0 +pytest==8.4.0 +httpx==0.28.1 +pydantic==2.11.5 +anyio==4.9.0 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000..bdc06e38 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +python3 - <<'PY' +from pathlib import Path +import secrets +root = Path('secrets') +p = root / 'postgres_password' +if not p.exists(): + p.write_text(secrets.token_hex(24)) +pw = p.read_text().strip() +for name, value in { + 'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger', + 'cache_url': 'redis://redis:6379/0', +}.items(): + path = root / name + if not path.exists(): + path.write_text(value) +# Bind-mounted Docker secrets must be readable by container uid 10001. +# Parent directory remains 0700 on the host. +for path in root.iterdir(): + path.chmod(0o444) +if not Path('.env').exists(): + Path('.env').write_text('SESSION_TTL_SECONDS=3600\n') +print('Local secrets ready (values hidden).') +PY diff --git a/scripts/capture_load.py b/scripts/capture_load.py new file mode 100644 index 00000000..1dea1d30 --- /dev/null +++ b/scripts/capture_load.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Capture repeatable short load run and sanitized environment/resource evidence.""" +import datetime +import json +import platform +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def output(args): + return subprocess.check_output(args, cwd=ROOT, text=True).strip() + + +def counts(): + code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions']})); c.close()" + return json.loads(output(['docker','compose','exec','-T','api','python','-c',code])) + + +def main(): + names = ['mtusi-lab2-api-1','mtusi-lab2-postgres-1','mtusi-lab2-redis-1'] + environment = { + 'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'host_os': platform.platform(), 'host_architecture': platform.machine(), + 'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])), + 'python_in_image': output(['docker','compose','exec','-T','api','python','--version']), + 'dataset_before': counts(), + 'limits': {'api':'1 CPU / 512 MiB','postgres':'1 CPU / 512 MiB','redis':'0.5 CPU / 128 MiB'}, + 'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.', + 'repetitions': 1, 'duration_seconds':30, 'concurrency':4, + } + if sys.platform == 'darwin': + environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string']) + environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize'])) + samples = [] + with (ROOT/'evidence/load-output.txt').open('w') as log: + proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT) + while proc.poll() is None: + rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names]) + samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')} + for row in rows.splitlines()]}) + time.sleep(1) + if proc.returncode: + raise SystemExit(proc.returncode) + environment['dataset_after'] = counts() + (ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n') + (ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n') + print((ROOT/'evidence/load.json').read_text()) + + +if __name__ == '__main__': + main() diff --git a/scripts/check_fresh_migrations.sh b/scripts/check_fresh_migrations.sh new file mode 100755 index 00000000..5004189c --- /dev/null +++ b/scripts/check_fresh_migrations.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# A unique disposable Compose project; never touches normal project volumes. +set -eu +cd "$(dirname "$0")/.." +project="mtusi-lab2-migration-check-$$" +if [ -n "$(docker compose -p "$project" ps -aq)" ]; then + echo 'Refusing to reuse a pre-existing scratch project' >&2 + exit 1 +fi +cleanup() { + docker compose -p "$project" down --volumes >/dev/null 2>&1 +} +trap cleanup EXIT INT TERM +docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis +docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache' +echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.' diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 00000000..72e30b69 --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Repeatable grade-4 index experiment: SQL plans on 10000 fresh synthetic messages.""" +import json +from uuid import uuid4 +import psycopg +from psycopg.rows import dict_row +from messenger.db import setting +from messenger.security import HASHER + + +def main(): + uid, cid = uuid4(), uuid4() + with psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row) as conn: + conn.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)', + (uid, 'plan_' + uuid4().hex[:24], 'Synthetic plan user', HASHER.hash(str(uuid4())))) + conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (cid, 'Synthetic 10000-message index experiment')) + conn.execute('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', (cid, uid)) + conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),%s,%s, 'Synthetic message '||n, timestamptz '2026-01-01T00:00:00Z' + n * interval '1 millisecond' FROM generate_series(1,10000) AS n", (cid, uid)) + with psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True) as conn: + conn.execute('ANALYZE messages') + # OFFSET is used only to choose an experimental pivot, never by the application pagination. + pivot = conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET 9000 LIMIT 1', (cid,)).fetchone() + result = {'messages_in_chat': 10000, 'limit_plus_one': 51, + 'index': 'messages_history_idx (chat_id, created_at, id)', + 'first_sql': 'SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=$1 ORDER BY created_at,id LIMIT 51', + 'deep_sql': 'SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=$1 AND (created_at,id)>($2,$3) ORDER BY created_at,id LIMIT 51', + 'first': None, 'deep_after_9001': None} + for key, condition, params in [ + ('first', '', (cid,)), + ('deep_after_9001', ' AND (created_at,id) > (%s,%s)', (cid, pivot['created_at'], pivot['id'])) + ]: + plan = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s'+condition+' ORDER BY created_at,id LIMIT 51', params).fetchone()['QUERY PLAN'][0] + result[key] = plan + nodes = [plan['Plan']] + index_found = False + while nodes: + node = nodes.pop() + index_found |= node.get('Index Name') == 'messages_history_idx' + nodes.extend(node.get('Plans', [])) + if not index_found: + raise RuntimeError('Planner did not use history index: ' + key) + print(json.dumps(result, indent=2, default=str)) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..2cfdc7e0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart api postgres redis +docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/verify_grade4.py b/scripts/verify_grade4.py new file mode 100644 index 00000000..f623fa78 --- /dev/null +++ b/scripts/verify_grade4.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Explicit local dependency fault experiment. Restores stopped services in finally.""" +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + + +def compose(*args): + return subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def wait_ready(c): + deadline = time.monotonic() + 60 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(0.5) + + +def main(): + c = Client(timeout=6) + actor = c.user('fault') + key = 'user:' + actor['id'] + compose('exec', '-T', 'redis', 'redis-cli', 'DEL', key) + c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + ttl = int(compose('exec', '-T', 'redis', 'redis-cli', 'TTL', key)) + check(0 < ttl <= 60, 'profile was not cached with TTL') + def hits(): + stats = compose('exec', '-T', 'redis', 'redis-cli', 'INFO', 'stats') + return int(next(line.split(':')[1] for line in stats.splitlines() if line.startswith('keyspace_hits:'))) + before = hits() + check(c.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'] == actor['user'], 'cached profile differs') + check(hits() > before, 'second profile read did not hit Redis') + results = {'cache_ttl_seconds': ttl, 'cache_miss_hit': 'PASS'} + try: + compose('stop', '-t', '2', 'redis') + response = c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + check(response['json'] == actor['user'] and response['elapsed'] < 3, 'Redis fallback failed or unbounded') + c.call('GET', '/health/ready', schema='Health') + c.call('GET', '/api/v1/users/me', 401) + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + c.call('GET', '/api/v1/users/me', 401, actor=actor) + results['redis_outage'] = {'result': 'PASS', 'fallback_seconds': round(response['elapsed'], 4), + 'readiness': 200, 'anonymous': 401, 'logout': 204, 'revoked': 401} + finally: + compose('start', 'redis') + # Restored cache contains the old public profile, but may never bypass session revocation. + c.call('GET', '/api/v1/users/me', 401, actor=actor) + actor = c.user('dbfault') + c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + try: + compose('stop', '-t', '2', 'postgres') + c.call('GET', '/health/live', schema='Health') + ready = c.call('GET', '/health/ready', 503) + response = c.call('GET', '/api/v1/users/me', 503, actor=actor) + check(response['elapsed'] < 5 and ready['elapsed'] < 5, 'Database timeout exceeded bound') + c.call('GET', '/api/v1/users/me', 401) + results['postgres_outage'] = {'result': 'PASS', 'authenticated_status':503, + 'response_seconds':round(response['elapsed'], 4), + 'readiness':503, 'liveness':200, 'anonymous':401} + finally: + compose('start', 'postgres') + recovered = time.monotonic() + wait_ready(c) + check(c.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'] == actor['user'], 'session lost after DB recovery') + results['postgres_recovery_seconds'] = round(time.monotonic()-recovered, 4) + results['result'] = 'PASS' + print(json.dumps(results, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_persistence.py b/scripts/verify_persistence.py new file mode 100644 index 00000000..7fd6e598 --- /dev/null +++ b/scripts/verify_persistence.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Public persistence test plus proof all three real containers restarted, volumes unchanged.""" +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def snapshot(): + ids = subprocess.check_output(['docker','compose','ps','-q'],cwd=ROOT,text=True).split() + data = json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)) + return {item['Config']['Labels']['com.docker.compose.service']: { + 'container': item['Name'].lstrip('/'), 'started_at': item['State']['StartedAt'], + 'image': item['Config']['Image'], + 'volumes': sorted(m['Name'] for m in item['Mounts'] if m['Type']=='volume') + } for item in data} + + +def main(): + before = snapshot() + result = subprocess.run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart.sh'],cwd=ROOT,capture_output=True,text=True) + after = snapshot() + print(result.stdout, end='') + if result.returncode: + print(result.stderr, file=sys.stderr) + raise SystemExit(result.returncode) + for service in ['api','postgres','redis']: + assert before[service]['started_at'] != after[service]['started_at'], service + ' did not restart' + assert before[service]['volumes'] == after[service]['volumes'], service + ' volumes changed' + evidence = {'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()} + path = ROOT / 'evidence' / 'persistence.json' + path.write_text(json.dumps(evidence, indent=2)+'\n') + print('PASS: actual API/PostgreSQL/Redis restart confirmed; volume identities unchanged.') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..6a5e3994 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,249 @@ +"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads.""" +import json +import logging +import re +from contextlib import asynccontextmanager +from uuid import uuid4 + +import psycopg +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from psycopg_pool import PoolTimeout, TooManyRequests +from redis.exceptions import RedisError +from starlette.exceptions import HTTPException +from starlette.concurrency import run_in_threadpool + +from .db import Database +from .errors import ApiError, invalid, not_found, unauthorized +from .models import CreateChat, CreateMessage, CreateUser, uuid_value +from .pagination import decode_cursor, page, page_limit, public, stamp +from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session + +LOG = logging.getLogger('messenger') +USER_FIELDS = 'id, username, display_name, created_at' + + +def error_response(request, error): + return JSONResponse({'error': {'code': error.code, 'message': error.message, + 'request_id': request.state.request_id}}, + status_code=error.status, headers=error.headers) + + +def authenticate(request: Request): + token = bearer(request.headers.get('Authorization', '')) + token_hash = digest(token) + with request.app.state.db.pool.connection() as conn: + row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', + (token_hash,)).fetchone() + if not row: + raise unauthorized() + request.state.token_hash = token_hash + return row['user_id'] + + +def require_member(conn, chat_id, user_id): + found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone() + if not found: + raise not_found() + + +def chat_public(conn, row): + item = public(row) + item['member_ids'] = [str(member['user_id']) for member in conn.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()] + return item + + +def create_app(): + @asynccontextmanager + async def lifespan(app): + app.state.db = Database() + app.state.db.open() + try: + yield + finally: + app.state.db.close() + + app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + @app.middleware('http') + async def request_context(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + path = request.url.path + json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path) + if request.method == 'POST' and json_endpoint: + media = request.headers.get('Content-Type', '').split(';') + unsupported = media[0].strip().lower() != 'application/json' + for parameter in media[1:]: + name, _, value = parameter.strip().partition('=') + if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'): + unsupported = True + if unsupported: + if path != '/api/v1/users': + await run_in_threadpool(authenticate, request) + raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8') + try: + json.loads((await request.body()).decode('utf-8')) + except (UnicodeError, json.JSONDecodeError): + raise invalid() from None + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc) + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests): + response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно')) + except Exception as exc: + # Log exception class only: credentials, SQL and message content stay out of logs. + LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__) + response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка')) + response.headers['X-Request-Id'] = request.state.request_id + return response + + @app.exception_handler(ApiError) + async def api_error(request, exc): + return error_response(request, exc) + + @app.exception_handler(RequestValidationError) + async def validation_error(request, exc): + return error_response(request, invalid()) + + @app.exception_handler(HTTPException) + async def http_error(request, exc): + return error_response(request, not_found() if exc.status_code == 404 else invalid()) + + @app.get('/health/live') + def live(): + return {'status': 'ok'} + + @app.get('/health/ready') + def ready(request: Request): + with request.app.state.db.pool.connection() as conn: + conn.execute('SELECT 1') + return {'status': 'ok'} + + @app.post('/api/v1/users', status_code=201) + def register(body: CreateUser, request: Request): + password_hash = HASHER.hash(body.password) + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute( + f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}', + (uuid4(), body.username, body.display_name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, 'conflict', 'Имя пользователя занято') from None + return public(row) + + @app.post('/api/v1/auth/sessions', status_code=201) + async def create_session(request: Request): + # Run hashing and synchronous database calls outside the event loop. + username, password = basic_credentials(request.headers.get('Authorization', '')) + if await request.body(): + raise invalid() + return await run_in_threadpool(login, request, username, password) + + def login(request, username, password): + db = request.app.state.db + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone() + valid = check_password(row['password_hash'] if row else DUMMY_HASH, password) + if not row or not valid: + raise unauthorized(basic=True) + row.pop('password_hash') + token, expires_at = new_session(db.session_ttl) + with db.pool.connection() as conn: + conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)', + (digest(token), row['id'], expires_at)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)} + + @app.get('/api/v1/users/me') + def me(request: Request, user_id=Depends(authenticate)): + db = request.app.state.db + key = 'user:' + str(user_id) + try: + cached = db.cache.get(key) + if cached: + return json.loads(cached) + except (RedisError, ValueError): + pass + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone() + if not row: + raise unauthorized() + result = public(row) + try: + db.cache.set(key, json.dumps(result), ex=60) + except RedisError: + pass + return result + + @app.delete('/api/v1/auth/sessions/current', status_code=204) + def logout(request: Request, user_id=Depends(authenticate)): + with request.app.state.db.pool.connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,)) + return Response(status_code=204) + + @app.post('/api/v1/chats', status_code=201) + def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)): + if str(user_id) not in body.member_ids: + raise invalid() + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at', + (uuid4(), body.title)).fetchone() + # An unknown user raises AFTER the chat INSERT; transaction context rolls it all back. + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', + [(row['id'], uuid_value(member)) for member in body.member_ids]) + result = chat_public(conn, row) + except psycopg.errors.ForeignKeyViolation: + raise invalid() from None + return result + + @app.get('/api/v1/chats') + def list_chats(request: Request, user_id=Depends(authenticate)): + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'chats:' + str(user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (c.created_at,c.id) > (%s,%s)' if after else '' + params = (user_id, *after, limit+1) if after else (user_id, limit+1) + with request.app.state.db.pool.connection() as conn: + rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall() + result = page(scope, rows, limit) + result['items'] = [chat_public(conn, row) for row in rows[:limit]] + return result + + @app.get('/api/v1/chats/{chat_id}') + def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone() + return chat_public(conn, row) + + @app.post('/api/v1/chats/{chat_id}/messages', status_code=201) + def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) ' + 'RETURNING id,chat_id,sender_id,text,created_at', + (uuid4(), cid, user_id, body.text)).fetchone() + return public(row) + + @app.get('/api/v1/chats/{chat_id}/messages') + def history(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'messages:' + str(cid) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (created_at,id) > (%s,%s)' if after else '' + params = (cid, *after, limit+1) if after else (cid, limit+1) + rows = conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s'+where+ + ' ORDER BY created_at,id LIMIT %s', params).fetchall() + return page(scope, rows, limit) + + return app diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..27356ec1 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,42 @@ +"""Bounded dependencies; PostgreSQL is always the source of truth.""" +import os +from pathlib import Path + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +def setting(name, default=None): + filename = os.environ.get(name + '_FILE') + value = Path(filename).read_text().strip() if filename else os.environ.get(name, default) + if value is None: + raise RuntimeError(f'Missing configuration: {name}') + return value + + +class Database: + def __init__(self): + self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600')) + if self.session_ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + self.pool = ConnectionPool( + setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')), + timeout=2, max_waiting=16, open=False, + kwargs={'row_factory': dict_row, 'connect_timeout': 3, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'}, + ) + self.cache = Redis.from_url( + setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8, + retry=Retry(NoBackoff(), 0), + ) + + def open(self): + self.pool.open(wait=True, timeout=30) + + def close(self): + self.pool.close() + self.cache.close() diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..082390e7 --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,19 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid(): + return ApiError(400, 'invalid_request', 'Неверный запрос') + + +def unauthorized(basic=False): + scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme}) + + +def not_found(): + return ApiError(404, 'not_found', 'Объект не найден') diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..e591e4ee --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,29 @@ +"""Versioned, atomic, repeatable migrations, serialized with an advisory lock.""" +import hashlib +from pathlib import Path + +import psycopg +from .db import setting + + +def main(): + directory = Path(__file__).resolve().parents[2] / 'migrations' + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(20261102)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())') + for path in sorted(directory.glob('*.sql')): + sql = path.read_text() + checksum = hashlib.sha256(sql.encode()).hexdigest() + found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone() + if found: + if found[0] != checksum: + raise RuntimeError('Applied migration checksum changed: ' + path.name) + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum)) + print('Applied', path.name) + print('Migrations up to date') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 00000000..4aeaf2ad --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,60 @@ +import re +from uuid import UUID +from pydantic import BaseModel, ConfigDict, Field, field_validator +from .errors import invalid + + +def uuid_value(value): + try: + # Require the UUID string representation, not integer/coercible values. + if not isinstance(value, str) or len(value) != 36: + raise ValueError + return UUID(value) + except (ValueError, TypeError, AttributeError): + raise invalid() from None + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra='forbid', strict=True) + + @field_validator('*') + @classmethod + def valid_strings(cls, value): + if isinstance(value, str): + if not value.strip(): + raise ValueError('Blank string') + # PostgreSQL cannot store NUL or unpaired Unicode surrogates. + if '\x00' in value: + raise ValueError('NUL is not supported') + value.encode('utf-8') + return value + + +class CreateUser(StrictModel): + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('username') + @classmethod + def username_ascii(cls, value): + if not re.fullmatch(r'[a-z0-9_]{3,32}', value): + raise ValueError('Invalid username') + return value + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator('member_ids') + @classmethod + def members(cls, value): + ids = [uuid_value(item) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('Duplicate member') + return [str(item) for item in ids] + + +class CreateMessage(StrictModel): + text: str = Field(min_length=1, max_length=4000) diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..0a875068 --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,48 @@ +import base64 +import json +import re +from datetime import datetime, timezone +from .errors import invalid +from .models import uuid_value + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def encode_cursor(scope, row): + raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip('=') + + +def decode_cursor(scope, value): + if value is None: + return None + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value): + raise ValueError + data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + raise ValueError + ts = datetime.fromisoformat(data[1].replace('Z', '+00:00')) + if ts.tzinfo is None: + raise ValueError + return ts, uuid_value(data[2]) + except (ValueError, TypeError, AttributeError, OverflowError): + raise invalid() from None + + +def page_limit(value): + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + raise invalid() + return int(value) + + +def public(row): + return {key: stamp(value) if isinstance(value, datetime) else str(value) if hasattr(value, 'hex') else value + for key, value in row.items()} + + +def page(scope, rows, limit): + return {'items': [public(row) for row in rows[:limit]], + 'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None} diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..b82c6b0c --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,49 @@ +import base64 +import hashlib +import re +import secrets +from datetime import datetime, timedelta, timezone +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError +from .errors import unauthorized + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash('synthetic-dummy-password') + + +def digest(token): + return hashlib.sha256(token.encode()).hexdigest() + + +def basic_credentials(header): + try: + scheme, encoded = header.split(' ', 1) + if scheme.lower() != 'basic': + raise ValueError + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128: + raise ValueError + return username, password + except (ValueError, UnicodeError): + raise unauthorized(basic=True) from None + + +def check_password(encoded, password): + try: + return HASHER.verify(encoded, password) + except (VerificationError, InvalidHashError): + return False + + +def bearer(header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError + return token + except ValueError: + raise unauthorized() from None + + +def new_session(ttl): + return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..39a8efea --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,141 @@ +"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities.""" +import base64 +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fastapi.testclient import TestClient +from messenger.app import create_app + +PASSWORD = 'Тестовый:пароль🔐' * 8 + + +@pytest.fixture +def client(): + with TestClient(create_app()) as c: + yield c + + +def user(c): + body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD) + r = c.post('/api/v1/users', json=body) + assert r.status_code == 201, r.text + return r.json(), body + + +def login(c, body): + credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode() + r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials}) + assert r.status_code == 201, r.text + return {'Authorization': 'Bearer ' + r.json()['token']}, r.json() + + +def test_auth_unicode_logout_and_cache(client): + u, body = user(client) + h, session = login(client, body) + assert session['user'] == u + assert len(session['token']) >= 43 + key = 'user:' + u['id'] + client.app.state.db.cache.delete(key) + for _ in range(2): + assert client.get('/api/v1/users/me', headers=h).json() == u + cached = json.loads(client.app.state.db.cache.get(key)) + assert cached == u + assert 0 < client.app.state.db.cache.ttl(key) <= 60 + assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204 + r = client.get('/api/v1/users/me', headers=h) + assert r.status_code == 401 + assert r.headers['www-authenticate'] == 'Bearer realm="messenger"' + assert r.json()['error']['request_id'] == r.headers['x-request-id'] + + +def test_session_expiry(monkeypatch): + monkeypatch.setenv('SESSION_TTL_SECONDS', '2') + with TestClient(create_app()) as client: + _, body = user(client) + h, _ = login(client, body) + assert client.get('/api/v1/users/me', headers=h).status_code == 200 + time.sleep(2.1) + assert client.get('/api/v1/users/me', headers=h).status_code == 401 + + +def test_registration_race(client): + body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD) + with ThreadPoolExecutor(max_workers=6) as executor: + codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6))) + assert sorted(codes) == [201, 409, 409, 409, 409, 409] + + +def test_chat_rollback_after_insert(client): + u, body = user(client) + h, _ = login(client, body) + title = 'rollback_' + uuid.uuid4().hex + r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]}) + assert r.status_code == 400 + with client.app.state.db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0 + + +def test_cursor_access_and_strict_validation(client): + a, ab = user(client) + b, bb = user(client) + ah, _ = login(client, ab) + bh, _ = login(client, bb) + chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)] + path = '/api/v1/chats/' + chats[0]['id'] + for text in ['Один 👋', 'Два', 'Три']: + assert client.post(path+'/messages', headers=ah, json={'text':text}).status_code == 201 + p = client.get(path+'/messages?limit=2', headers=ah).json() + assert [m['text'] for m in p['items']] == ['Один 👋','Два'] + tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json() + assert [m['text'] for m in tail['items']] == ['Три'] + assert tail['next_cursor'] is None + r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']}) + assert r.status_code == 400 + assert client.get(path, headers=bh).status_code == 404 + for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]: + assert client.post(path+'/messages',headers=ah,json=payload).status_code == 400 + for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']: + assert client.get(path+'/messages?'+query,headers=ah).status_code == 400 + for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]: + assert client.post('/api/v1/users',json=payload).status_code == 400 + assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400 + + +def test_session_storage_and_migrations(client): + from messenger.migrate import main as migrate + u, body = user(client) + _, session = login(client, body) + db = client.app.state.db + def snapshot(): + with db.pool.connection() as conn: + return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] + for table in ['users','chats','chat_members','messages','sessions','schema_migrations']} + before = snapshot() + migrate() + migrate() + assert snapshot() == before + with db.pool.connection() as conn: + row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone() + assert row['token_hash'] != session['token'] + assert len(row['token_hash']) == 64 + stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash'] + assert stored.startswith('$argon2id$') + assert stored != body['password'] + + +def test_json_media_type_and_utf8(client): + body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD) + raw = json.dumps(body).encode() + for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]: + r = client.post('/api/v1/users',content=raw,headers=headers) + assert r.status_code == 415 + assert r.json()['error']['code'] == 'unsupported_media_type' + for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']: + r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'}) + assert r.status_code == 400, encoding + r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'}) + assert r.status_code == 401 diff --git a/tests/client.py b/tests/client.py index bf469357..f0499a29 100644 --- a/tests/client.py +++ b/tests/client.py @@ -11,7 +11,7 @@ import time import urllib.error import urllib.request import uuid -# + ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) -- 2.54.0