diff --git a/.dockerignore b/.dockerignore index bc16e556..6b0b11b1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,20 @@ +# Keep the image minimal: only src/, migrations/ and requirements.txt are copied. .git .github +.gitignore +.gitattributes +.dockerignore +Dockerfile +Dockerfile.example +compose*.yaml +compose*.yml +stack*.yaml + +# Secrets / local state (never bake these in). .env .env.* -!.env.example secrets -certs +secrets.example volumes data backups @@ -13,8 +23,31 @@ backups *.p12 *.pfx *.jks + +# Local tooling and caches. .venv +venv __pycache__ +*.py[cod] node_modules +.pytest_cache +.mypy_cache +.ruff_cache +*.log + +# Tests, evidence and docs are not needed to run the API. +tests +student-tests evidence +examples +contracts +scripts +infra +*.md +lab.json +Makefile + +# Editor / OS noise. +.idea +.vscode .DS_Store diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..cf6c9216 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Keep shell hooks LF so they stay executable on Linux/CI. +*.sh text eol=lf diff --git a/.gitignore b/.gitignore index b5a71896..df2af196 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,7 @@ coverage/ .DS_Store .idea/ .vscode/ +# Personal study notes, not part of the submission. +EXPLAIN.md # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c8f60042 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 +# Lab3 messenger image. Python is interpreted, so one pinned stage is enough. +# Migrations run from a separate one-shot compose service using the same image. +FROM python:3.11.9-slim-bookworm + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt + +COPY src ./src +COPY migrations ./migrations + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=5 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +CMD ["python", "-m", "src.main"] diff --git a/Makefile b/Makefile index 5d86d171..26cfe223 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-student help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; test-student: own pytest (grade 4); load: synthetic writes" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +28,6 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + $(PYTHON) -m pytest student-tests -q diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..417a0cc3 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,129 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Журавский Егор БВТ2301 (заполняет студент) +- Целевая оценка; выбранный трек 5, если нужен: **4** (уровень 3 + уровень 4); трек 5 не заявлен +- Commit/PR: TODO после публикации (ветка `solution/БВТ2301-Журавский`) +- Использованные источники, библиотеки и помощь ИИ: FastAPI 0.115.6, Pydantic 2.10.6, Uvicorn 0.34.0, psycopg 3.2.3 + psycopg-pool 3.2.4, argon2-cffi 23.1.0, redis-py 5.2.1 (клиент Valkey), **boto3 1.35.90**, **Pillow 11.1.0**; pytest и openapi-spec-validator только для проверок. Инфраструктура: nginx 1.27.4, PostgreSQL 16.6, Valkey 8.0.2, SeaweedFS 3.80. Документация S3/SeaweedFS, Pillow, RFC 7617. Помощь ИИ: генерация каркаса и текста, проверено запуском тестов. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Версии и prerequisites** + +| Компонент | Версия / образ | +| --- | --- | +| Runtime | Python 3.11.9 | +| API-образ | `python:3.11.9-slim-bookworm` (пин; digest `sha256:8fb099…c317`) | +| Балансировщик | `nginx:1.27.4-alpine` | +| PostgreSQL | `postgres:16.6-bookworm` | +| Valkey | `valkey/valkey:8.0.2-alpine` | +| S3 | `chrislusf/seaweedfs:3.80` (S3-совместимый, приватный bucket) | +| Python-зависимости | `requirements.txt` (fastapi, pydantic, uvicorn, psycopg[binary], psycopg-pool, argon2-cffi, redis, boto3, pillow) | +| Стенд | Windows 10 Pro 19045, Ryzen 5 3600 (6C/12T), 15.9 GiB RAM; хуки/Compose выполняются в WSL Ubuntu (или Linux) | + +Prerequisites: Docker с Linux-контейнерами и плагином Compose. POSIX-shell (WSL/Git Bash/Linux) нужен для `scripts/*.sh` и публичных `test-persistence`/`test-failover`. + +**Создание локальных secrets (не коммитятся)** + +```sh +sh scripts/gen-secrets.sh +# создаёт ./secrets/{database_password,database_url,cache_url,s3_root_*,s3_*,s3_config.json} +# либо: cp secrets.example/* secrets/ и вписать значения +``` + +**Запуск, bootstrap, проверка, остановка** + +```sh +docker compose --env-file .env config --quiet # 1. валидация +docker compose --env-file .env up -d --build # 2. api×2 + lb + postgres + valkey + s3 (+ migrate, s3-init) +docker compose ps # 3. дождаться healthy у api/lb/postgres/valkey/s3 + +# migrate и s3-init — одноразовые сервисы: применяют схему и создают приватный bucket +# (идемпотентны, повторный запуск не дублирует данные). + +python tests/smoke.py --base-url http://localhost:8080 # 4. публичный smoke +.venv\Scripts\python -m pytest student-tests -v # свои тесты (см. ниже) + +make test-persistence ACTION_SCRIPT=scripts/restart.sh # в WSL/Linux +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # в WSL/Linux (после — restore-api.sh) +python scripts/check-s3-outage.py # отказ S3 -> 503 и восстановление +docker compose exec -T api python -m src.reconcile # dry-run сверки S3<->БД +docker compose exec -T api python -m src.s3_private_check # приватность bucket вне API + +# Безопасная остановка (данные сохраняются): +docker compose stop + +# Намеренное удаление учебных данных (тома PostgreSQL, Valkey и S3): +docker compose down -v +``` + +**Роль → service → версия → health → volume → порт → credentials** + +| Роль | service | версия | healthcheck | volume | порт | credentials | +| --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | nginx 1.27.4-alpine | `GET /health/live` | — | публикует `127.0.0.1:8080` | — | +| api | `api` ×2 | из исходников (`messenger-api:lab4`) | `GET /health/ready` (PG+S3) | — | только внутренний `expose:8080` | секреты файлами | +| postgres | `postgres` | 16.6-bookworm | `pg_isready` | `pgdata` | внутренний | `database_password` | +| valkey | `valkey` | 8.0.2-alpine | `valkey-cli ping` | `valkeydata` | внутренний | — | +| s3 | `s3` | seaweedfs 3.80 | `master/cluster/status` | `s3data` | внутренний `expose:8333` | `s3_config` (root+api) | +| bootstrap | `s3-init` | из исходников | — | — | — | `s3_root_*` | +| migrate | `migrate` | из исходников | — | — | — | `database_url` | ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема:** `Клиент → nginx LB → API ×2 → PostgreSQL (метаданные) + Valkey (кэш сессий) + SeaweedFS S3 (байты)`. Публикуется только порт балансировщика; реплики API и S3 доступны лишь внутри compose-сети. + +- **Метаданные — PostgreSQL, байты — S3.** Таблица `attachments` (+ `message_attachments`) хранит `s3_key`, `content_type`, `size_bytes`, `sha256`, `status`, `error_code`. Ключ объекта **назначает сервер** (`chats/{chat_id}/{attachment_id}`), `filename` — только для отображения. Bucket приватный, у API отдельные ограниченные S3-креды (identity `messenger-api`, действия `Read/Write/List:`); root используется только одноразовым `s3-init`. +- **Транзакционная граница двух хранилищ.** Синхронный upload: поток → временный spill-to-disk файл (жёсткий лимит байт) → проверка изображения → `S3 PUT` → `INSERT` метаданных в одной транзакции. Если COMMIT не удался после PUT, объект удаляется best-effort (компенсация). Окно падения (краш между PUT и COMMIT) закрывает сверка `src.reconcile` с grace period. +- **Проверка изображений по содержимому.** Формат и размеры определяет Pillow по байтам, а не `Content-Type`/`filename`. Предел пикселей проверяется по заголовку до полного декодирования (защита от decompression bomb). >10 MiB → 413, не PNG/JPEG или повреждённый → 415, опасный `filename` → 400. +- **Доступ.** Любой участник чата читает метаданные и байты вложения; посторонний получает 404 (не 403). Привязать к сообщению можно только вложение **этого** чата в статусе `ready`. +- **Синхронный 201.** Lab4 возвращает 201 `ready` сразу после сохранения оригинала и метаданных (асинхронный режим — с lab5). +- **Масштабирование (перенос lab3).** Балансировщик, ≥2 взаимозаменяемые реплики, общие PostgreSQL/Valkey, `X-Instance-Id`, идемпотентность сообщений через общую таблицу `idempotency_keys` + advisory-lock. Хэш запроса сообщения теперь включает `attachment_ids` (отсутствие ≡ `[]`, порядок не значим). +- **Отказоустойчивость зависимостей.** PostgreSQL и S3 критичны для синхронного пути: `/health/ready` = 503 при недоступности любого из них. Недоступность Valkey обходится (кэш сессий не источник истины). Каждая операция S3/Valkey ограничена таймаутом и mini circuit breaker, т.к. DNS остановленного контейнера может блокироваться. + +**Соответствие `.env.example`** + +| Переменная | Роль | +| --- | --- | +| `APP_HOST`, `APP_PORT`, `APP_ENV` | адрес/порт Uvicorn, метка среды | +| `SESSION_TTL_SECONDS`, `SESSION_STORE`, `SESSION_CACHE_TTL_SECONDS` | срок сессии и кэша Valkey | +| `IDEMPOTENCY_TTL_SECONDS` | срок хранения ключей идемпотентности | +| `DATABASE_URL_FILE`, `CACHE_URL_FILE` | файлы с DSN PostgreSQL и URL Valkey | +| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | адрес/регион/имя приватного bucket | +| `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` | ограниченные S3-креды API | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | лимиты загрузки (10 MiB, 40 млн px) | +| `API_REPLICAS` | число реплик API | ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke (уровень 3) | `python tests/smoke.py --base-url http://localhost:8080` | `PASS: lab4 public API scenario` | `evidence/lab4-smoke.txt` | +| Кол-во реплик, балансировка, `X-Instance-Id`, идемпотентность (перенос lab3) | `student-tests/test_instances.py`, `test_idempotency.py` | 2 реплики, replay ≡ body | `evidence/lab4-student-tests.txt` | +| S3 в стенде; метаданные в PostgreSQL, байты в приватном bucket; 201 ready; побайтный download участником; 404 постороннему | `test_attachments.py` + smoke | roundtrip sha совпадает, чужой → 404 | `evidence/lab4-student-tests.txt`, `evidence/lab4-smoke.txt` | +| PNG/JPEG ≤10 MiB и ≤40 млн px; MIME по содержимому; опасный filename; лимиты до декодирования | `test_attachments.py` | 413 (размер), 415 (corrupt/spoof), 400 (filename) | `evidence/lab4-student-tests.txt` | +| Upload → остановка реплики → download через другую; restart сохраняет метаданные и оригинал | `make test-persistence`, `make test-failover` | persistence PASS; failover: 2 инстанса, recovery 0.89 c, вложение сохранено | `evidence/lab4-persistence.txt`, `evidence/lab4-failover.txt` | +| Уровень 4: тесты слишком большого/повреждённого/подменённого файла, чужого attachment_id и файла другого чата; пользовательская ошибка ≠ 500 | `test_attachments.py` | 413/415/404, ни одного 500 | `evidence/lab4-student-tests.txt` | +| Уровень 4: компенсация/сверка orphan/missing с grace period | `docker compose exec -T api python -m src.reconcile [--apply]` | orphan найден, при grace не удалён, при `--grace-seconds 0` удалён, state сошёлся | `evidence/lab4-s3-reconcile.txt` | +| Уровень 4: ≥10 одновременных загрузок с измерением пика памяти | `python3 scripts/check-upload-load.py --uploads 12 --megabytes 6` | 12/12 ok; baseline ~177 МБ, пик 388.7 МБ (2 реплики) | `evidence/lab4-upload-load.txt` | +| Уровень 4: идемпотентный bootstrap bucket; ограниченные S3-креды; консоль не опубликована; приватность вне API | `s3-init`, `scripts/gen-secrets.sh`, `src.s3_private_check` | bucket создаётся идемпотентно; anonymous GET → 403 | `evidence/lab4-s3-private.txt`, `secrets.example/README.md` | +| Отказ внешнего хранилища не роняет API | `scripts/check-s3-outage.py` | S3 down → ready 503, upload 503; метаданные 200; после старта upload 201 | `evidence/lab4-s3-outage.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: байты в приватном S3, метаданные в PostgreSQL; рестарт и потеря одной API-реплики не теряют вложения, а отказ S3 деградирует ограниченно. Методика: реальный compose-стенд, синтетические PNG, хуки в WSL. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker compose restart api lb postgres valkey s3` | метаданные, сессия и оригинал сохранены | `PASS: persistence preserved confirmed data and existing session`; volumes на месте | секунды; потерь нет | +| Стоп одной API-реплики под нагрузкой | записи продолжаются через LB, реплика не возвращается | 2 инстанса до; 137 записей, 0 ошибок; recovery 0.89 c; вложение читается | ~0.9 c | +| Остановка S3 | bounded 503, метаданные доступны | ready 503, upload 503, `/users/me` 200; после старта upload 201 | восстановление после старта S3 | +| 12 параллельных загрузок по ~6 MiB | все успешны, память ограничена | 12/12 ok за 4.27 c; пик API 388.7 МБ на 2 реплики | n/a | +| «Краш» между PUT и COMMIT (orphan) | обнаружение и безопасная очистка с grace | dry-run не удаляет свежий orphan; `--apply --grace-seconds 0` удаляет 1 | n/a | +| Anonymous GET объекта в S3 (вне API) | отказ | HTTP 403 | n/a | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается: рестарт API/хранилищ с volumes, потеря одной API-реплики, потеря Valkey, ограниченный отказ S3 (503) с восстановлением, некорректные файлы, гонки/дубли. +- **Не** выдерживается и не заявлено: потеря Docker-host, отказ единственного S3/PostgreSQL (нет HA), фоновая обработка и варианты вложений (lab5), TLS (lab7). Балансировщик и БД — единичные точки отказа. +- Учебные упрощения: S3 bootstrap и схема применяются одноразовыми compose-сервисами; сверка S3↔БД запускается вручную (для lab5 появится worker); rate limiting и лимиты очереди не реализованы. +- Подтверждено измерениями: побайтный roundtrip, приватность bucket, ограниченная деградация при отказе S3, память при параллельных загрузках. Гипотеза на будущее — согласованность при асинхронной обработке (трек 5 следующей лабы). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..3f594652 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,210 @@ +# Lab4 topology: nginx LB -> 2+ interchangeable API replicas -> shared +# PostgreSQL (metadata, source of truth) + Valkey (session cache) + a private +# S3-compatible store (SeaweedFS) for attachment bytes. No API replica or S3 +# port is published, so the only reachable path is through the balancer. +# Secrets are file-based and live in ./secrets (gitignored). Schema migrations +# and the bucket bootstrap run in one-shot services before replicas start. +name: messenger-lab4 + +x-api-env: &api-env + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: ${SESSION_STORE:-valkey} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + CACHE_INVALIDATION_INTERVAL_SECONDS: ${CACHE_INVALIDATION_INTERVAL_SECONDS:-0.5} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT: ${DB_POOL_TIMEOUT:-5} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + +services: + lb: + image: nginx:1.27.4-alpine + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:${PUBLISH_PORT:-8080}:8080" + depends_on: + - api + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live >/dev/null 2>&1 || exit 1"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + + api: + build: + context: . + image: messenger-api:lab4 + environment: *api-env + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 20s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.75" + memory: 256M + + # One-shot schema migration; runs once before replicas start. + migrate: + build: + context: . + image: messenger-api:lab4 + command: ["python", "-m", "src.migrate"] + environment: *api-env + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + + # One-shot, idempotent bucket bootstrap with the root credentials. + s3-init: + build: + context: . + image: messenger-api:lab4 + command: ["python", "-m", "src.s3_bootstrap"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id + S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key + secrets: + - s3_root_access_key_id + - s3_root_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + s3: + image: chrislusf/seaweedfs:3.80 + command: + - server + - -dir=/data + - -ip=0.0.0.0 + - -s3 + - -s3.port=8333 + - -s3.config=/run/secrets/s3_config + volumes: + - s3data:/data + secrets: + - s3_config + expose: + - "8333" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 15s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + postgres: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/database_password + secrets: + - database_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + valkey: + image: valkey/valkey:8.0.2-alpine + command: ["valkey-server", "--appendonly", "yes"] + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + +volumes: + pgdata: + valkeydata: + s3data: + +secrets: + database_url: + file: ./secrets/database_url + database_password: + file: ./secrets/database_password + cache_url: + file: ./secrets/cache_url + s3_config: + file: ./secrets/s3_config.json + s3_root_access_key_id: + file: ./secrets/s3_root_access_key_id + s3_root_secret_access_key: + file: ./secrets/s3_root_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/evidence/lab4-failover.txt b/evidence/lab4-failover.txt new file mode 100644 index 00000000..5a262aa8 --- /dev/null +++ b/evidence/lab4-failover.txt @@ -0,0 +1,35 @@ +# Failover: stop ONE API replica under load, verified recovery via the LB +## containers before +messenger-lab4-api-1 Up About a minute (healthy) +messenger-lab4-api-2 Up About a minute (healthy) + +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 137, "failed_attempts": 0, "first_recovery_seconds": 0.89} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-migrate-1 Created + Container messenger-lab4-s3-1 Running + Container messenger-lab4-valkey-1 Running + Container messenger-lab4-s3-init-1 Created + Container messenger-lab4-api-1 Created + Container messenger-lab4-api-2 Running + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-valkey-1 Waiting + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-valkey-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-1 Starting + Container messenger-lab4-api-1 Started + +## containers after restore +messenger-lab4-api-1 Up Less than a second (health: starting) +messenger-lab4-api-2 Up About a minute (healthy) diff --git a/evidence/lab4-persistence.txt b/evidence/lab4-persistence.txt new file mode 100644 index 00000000..ec2c0ab8 --- /dev/null +++ b/evidence/lab4-persistence.txt @@ -0,0 +1,24 @@ +# Persistence: restart API replicas + lb + PostgreSQL + Valkey + S3 (volumes kept) +## containers before +messenger-lab4-api-1 Up 2 minutes (healthy) +messenger-lab4-api-2 Up 2 minutes (healthy) +messenger-lab4-lb-1 Up 5 minutes (healthy) +messenger-lab4-postgres-1 Up 5 minutes (healthy) +messenger-lab4-s3-1 Up 54 seconds (healthy) +messenger-lab4-valkey-1 Up 5 minutes (healthy) + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +## containers after +messenger-lab4-api-1 Up 32 seconds (healthy) +messenger-lab4-api-2 Up 32 seconds (healthy) +messenger-lab4-lb-1 Up 32 seconds (health: starting) +messenger-lab4-postgres-1 Up 32 seconds (healthy) +messenger-lab4-s3-1 Up 24 seconds (healthy) +messenger-lab4-valkey-1 Up 33 seconds (healthy) + +## volumes still present +messenger-lab4_pgdata +messenger-lab4_s3data +messenger-lab4_valkeydata diff --git a/evidence/lab4-s3-outage.txt b/evidence/lab4-s3-outage.txt new file mode 100644 index 00000000..a0c01b24 --- /dev/null +++ b/evidence/lab4-s3-outage.txt @@ -0,0 +1,10 @@ + Container messenger-lab4-s3-1 Stopping + Container messenger-lab4-s3-1 Stopped + Container messenger-lab4-s3-1 Starting + Container messenger-lab4-s3-1 Started +baseline: upload = 201 +stopping s3 ... +s3 down: /health/ready = 503, upload = 503 (bounded) +s3 down: /users/me = 200 (metadata unaffected) +s3 restored: upload = 201 +PASS: S3 outage degrades to bounded 503 and recovers diff --git a/evidence/lab4-s3-private.txt b/evidence/lab4-s3-private.txt new file mode 100644 index 00000000..559d74e9 --- /dev/null +++ b/evidence/lab4-s3-private.txt @@ -0,0 +1,2 @@ +anonymous GET -> 403 (denied) +PASS: private bucket denies anonymous access diff --git a/evidence/lab4-s3-reconcile.txt b/evidence/lab4-s3-reconcile.txt new file mode 100644 index 00000000..f7eaa111 --- /dev/null +++ b/evidence/lab4-s3-reconcile.txt @@ -0,0 +1,13 @@ +# S3/DB consistency: orphan detection with a safe grace period, then cleanup +== baseline == +{"db_attachments": 44, "s3_objects": 44, "orphans": 0, "missing": 0, "apply": false, "grace_seconds": 3600} +== create an orphan object (simulated crash between PUT and COMMIT) == +orphan put +== reconcile dry-run (fresh orphan kept by grace period) == +{"db_attachments": 44, "s3_objects": 45, "orphans": 0, "missing": 0, "apply": false, "grace_seconds": 3600} +== reconcile --apply --grace-seconds 0 == +{"db_attachments": 44, "s3_objects": 45, "orphans": 1, "missing": 0, "apply": true, "grace_seconds": 0} +orphan: chats/orphan/demo +{"deleted_orphans": 1} +== reconcile after cleanup == +{"db_attachments": 44, "s3_objects": 44, "orphans": 0, "missing": 0, "apply": false, "grace_seconds": 3600} diff --git a/evidence/lab4-smoke.txt b/evidence/lab4-smoke.txt new file mode 100644 index 00000000..167bf09c --- /dev/null +++ b/evidence/lab4-smoke.txt @@ -0,0 +1,3 @@ +# Public acceptance scenario (lab4) +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab4-student-tests.txt b/evidence/lab4-student-tests.txt new file mode 100644 index 00000000..d2a5e54f --- /dev/null +++ b/evidence/lab4-student-tests.txt @@ -0,0 +1,58 @@ +# Student black-box tests (grade 4), one command +$ C:\IT-labs\VNP\mtusi2026lab4\.venv\Scripts\python.exe -m pytest student-tests -v +============================= test session starts ============================= +platform win32 -- Python 3.11.9, pytest-8.3.4, pluggy-1.7.0 -- C:\IT-labs\VNP\mtusi2026lab4\.venv\Scripts\python.exe +cachedir: .pytest_cache +rootdir: C:\IT-labs\VNP\mtusi2026lab4 +plugins: anyio-4.16.0 +collecting ... collected 47 items + +student-tests/test_attachments.py::test_upload_download_roundtrip PASSED [ 2%] +student-tests/test_attachments.py::test_foreign_actor_gets_404 PASSED [ 4%] +student-tests/test_attachments.py::test_cross_chat_attachment_is_404 PASSED [ 6%] +student-tests/test_attachments.py::test_oversized_upload_is_413 PASSED [ 8%] +student-tests/test_attachments.py::test_corrupt_image_is_415 PASSED [ 10%] +student-tests/test_attachments.py::test_mime_spoof_is_415 PASSED [ 12%] +student-tests/test_attachments.py::test_empty_upload_is_415 PASSED [ 14%] +student-tests/test_attachments.py::test_unsafe_filename_is_400 PASSED [ 17%] +student-tests/test_attachments.py::test_empty_text_without_attachment_is_400 PASSED [ 19%] +student-tests/test_attachments.py::test_message_idempotency_includes_attachments PASSED [ 21%] +student-tests/test_auth.py::test_registration_requires_password PASSED [ 23%] +student-tests/test_auth.py::test_short_password_is_400 PASSED [ 25%] +student-tests/test_auth.py::test_extra_field_is_400 PASSED [ 27%] +student-tests/test_auth.py::test_malformed_json_is_400 PASSED [ 29%] +student-tests/test_auth.py::test_duplicate_username_is_409 PASSED [ 31%] +student-tests/test_auth.py::test_registration_never_returns_password PASSED [ 34%] +student-tests/test_auth.py::test_wrong_password_is_401_basic PASSED [ 36%] +student-tests/test_auth.py::test_unknown_user_is_401_basic PASSED [ 38%] +student-tests/test_auth.py::test_basic_on_business_endpoint_is_401_bearer PASSED [ 40%] +student-tests/test_auth.py::test_missing_auth_is_401_bearer PASSED [ 42%] +student-tests/test_auth.py::test_unknown_bearer_is_401 PASSED [ 44%] +student-tests/test_auth.py::test_x_user_id_is_ignored PASSED [ 46%] +student-tests/test_auth.py::test_logout_revokes_session PASSED [ 48%] +student-tests/test_concurrency.py::test_100_concurrent_messages_are_not_lost PASSED [ 51%] +student-tests/test_idempotency.py::test_replay_same_key_returns_same_message PASSED [ 53%] +student-tests/test_idempotency.py::test_same_key_different_body_is_409 PASSED [ 55%] +student-tests/test_idempotency.py::test_missing_key_is_400 PASSED [ 57%] +student-tests/test_idempotency.py::test_invalid_key_is_400 PASSED [ 59%] +student-tests/test_idempotency.py::test_concurrent_same_key_creates_exactly_one_message PASSED [ 61%] +student-tests/test_instances.py::test_instance_header_present_on_success PASSED [ 63%] +student-tests/test_instances.py::test_instance_header_present_on_error PASSED [ 65%] +student-tests/test_instances.py::test_at_least_two_instances_seen_through_lb PASSED [ 68%] +student-tests/test_negative.py::test_foreign_chat_is_404 PASSED [ 70%] +student-tests/test_negative.py::test_empty_message_is_400 PASSED [ 72%] +student-tests/test_negative.py::test_oversized_message_is_400 PASSED [ 74%] +student-tests/test_negative.py::test_limit_out_of_range_is_400 PASSED [ 76%] +student-tests/test_negative.py::test_sender_id_in_body_is_rejected PASSED [ 78%] +student-tests/test_negative.py::test_chat_without_caller_is_400 PASSED [ 80%] +student-tests/test_negative.py::test_chat_with_unknown_member_is_400 PASSED [ 82%] +student-tests/test_negative.py::test_chat_with_duplicate_members_is_400 PASSED [ 85%] +student-tests/test_pagination.py::test_full_traversal_without_duplicates PASSED [ 87%] +student-tests/test_pagination.py::test_cursor_from_other_chat_is_400 PASSED [ 89%] +student-tests/test_pagination.py::test_garbage_cursor_is_400 PASSED [ 91%] +student-tests/test_pagination.py::test_well_formed_cursor_with_wrong_types_is_400 PASSED [ 93%] +student-tests/test_race.py::test_concurrent_duplicate_username_single_winner PASSED [ 95%] +student-tests/test_race.py::test_failed_chat_creation_rolls_back PASSED [ 97%] +student-tests/test_session_revocation.py::test_logout_invalidates_across_replicas PASSED [100%] + +============================= 47 passed in 5.57s ============================== diff --git a/evidence/lab4-upload-load.txt b/evidence/lab4-upload-load.txt new file mode 100644 index 00000000..b1b8f789 --- /dev/null +++ b/evidence/lab4-upload-load.txt @@ -0,0 +1,6 @@ +# 12 concurrent uploads of ~6 MiB through the balancer; API memory sampled +## baseline (idle) +messenger-lab4-api-2 98.11MiB / 256MiB +messenger-lab4-api-1 79.3MiB / 256MiB +{"file_bytes": 6292108, "uploads": 12} +{"ok": 12, "errors": [], "seconds": 4.27, "api_peak_mem_mb": 388.7} diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..955e158b --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,62 @@ +# Lab4 edge load balancer. +# +# Docker's embedded DNS (127.0.0.11) returns one A record per running `api` +# replica. `resolve` + a shared `zone` make nginx re-resolve that name every +# `valid` seconds, so replicas that appear/disappear are picked up without a +# reload. Passive checks (max_fails/fail_timeout) eject a replica that refuses +# connections; proxy_next_upstream retries the same request on a healthy peer. +worker_processes auto; + +events { + worker_connections 2048; +} + +http { + resolver 127.0.0.11 valid=5s ipv6=off; + resolver_timeout 2s; + + access_log /dev/stdout; + error_log /dev/stderr warn; + + sendfile on; + tcp_nopush on; + keepalive_timeout 30; + + upstream api_backend { + zone api_backend 64k; + server api:8080 resolve max_fails=1 fail_timeout=5s; + } + + server { + listen 8080; + server_name _; + + # Allow one attachment (<= 10 MiB) plus multipart-free JSON overhead. + client_max_body_size 12m; + + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + + proxy_connect_timeout 2s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + + # Retry a request on another replica if a backend is down mid-flight. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + location /health/live { + access_log off; + proxy_pass http://api_backend; + } + + location / { + proxy_pass http://api_backend; + } + } +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..41499537 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,42 @@ +-- Lab2 initial schema. Idempotent and safe to re-run via the migration runner. +create table if not exists users ( + id uuid primary key, + username text not null unique, + display_name text not null, + password_hash text not null, + created_at timestamptz not null +); + +create table if not exists chats ( + id uuid primary key, + title text not null, + created_at timestamptz not null +); + +create table if not exists chat_members ( + chat_id uuid not null references chats (id) on delete cascade, + user_id uuid not null references users (id) on delete cascade, + primary key (chat_id, user_id) +); + +create table if not exists messages ( + id uuid primary key, + chat_id uuid not null references chats (id) on delete cascade, + sender_id uuid not null references users (id), + text text not null, + created_at timestamptz not null +); + +-- Composite index prepared for keyset pagination: +-- where chat_id = $1 and (created_at, id) > ($2, $3) order by created_at, id +create index if not exists messages_chat_created_id_idx + on messages (chat_id, created_at, id); + +create table if not exists sessions ( + token_hash text primary key, + user_id uuid not null references users (id) on delete cascade, + created_at timestamptz not null, + expires_at timestamptz not null +); + +create index if not exists sessions_expires_at_idx on sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..2fba4094 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,19 @@ +-- Lab3 shared idempotency ledger for POST message retries. +-- +-- Scope is (user_id, HTTP method + chat path, key); the stored row carries the +-- canonical request hash and the exact 201 body so any replica can replay the +-- same Message. Rows expire after IDEMPOTENCY_TTL_SECONDS (default 24h). +create table if not exists idempotency_keys ( + user_id uuid not null references users (id) on delete cascade, + scope text not null, + key text not null, + request_hash text not null, + status_code integer not null, + response_body jsonb not null, + created_at timestamptz not null default now(), + expires_at timestamptz not null, + primary key (user_id, scope, key) +); + +create index if not exists idempotency_keys_expires_at_idx + on idempotency_keys (expires_at); diff --git a/migrations/003_cache_invalidations.sql b/migrations/003_cache_invalidations.sql new file mode 100644 index 00000000..3b7d274c --- /dev/null +++ b/migrations/003_cache_invalidations.sql @@ -0,0 +1,15 @@ +-- Lab3 durable cache invalidation queue. +-- +-- Logout deletes the authoritative session row in PostgreSQL and enqueues the +-- token hash here in the SAME transaction. A lightweight background drainer in +-- each API replica removes the corresponding Valkey cache key and then the row. +-- If Valkey was unreachable at logout time, the row survives and the stale +-- cache entry is removed as soon as the cache comes back, instead of being +-- trusted until its TTL. +create table if not exists cache_invalidations ( + token_hash text primary key, + created_at timestamptz not null default now() +); + +create index if not exists cache_invalidations_created_at_idx + on cache_invalidations (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..0d3dce7d --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,31 @@ +-- Lab4 attachments: metadata in PostgreSQL, bytes in S3. +-- +-- ``s3_key`` is server-generated and never exposed through the API. Linkage to +-- a message is a many-to-many table so a message can carry ordered attachments +-- and the same attachment can be referenced again without duplicating bytes. +create table if not exists attachments ( + id uuid primary key, + chat_id uuid not null references chats (id) on delete cascade, + owner_id uuid not null references users (id), + filename text not null, + content_type text not null, + size_bytes bigint not null, + sha256 text not null, + status text not null, + s3_key text not null, + error_code text, + created_at timestamptz not null, + updated_at timestamptz not null +); + +create index if not exists attachments_chat_id_idx on attachments (chat_id); + +create table if not exists message_attachments ( + message_id uuid not null references messages (id) on delete cascade, + attachment_id uuid not null references attachments (id) on delete cascade, + position integer not null, + primary key (message_id, attachment_id) +); + +create index if not exists message_attachments_attachment_idx + on message_attachments (attachment_id); diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..a7643f94 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,4 @@ # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +# Student-owned black-box tests (grade 4). +pytest==8.3.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e8e3b4c2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# Runtime dependencies for the lab4 messenger. Pinned for reproducibility. +fastapi==0.115.6 +pydantic==2.10.6 +uvicorn[standard]==0.34.0 +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +argon2-cffi==23.1.0 +redis==5.2.1 +boto3==1.35.90 +pillow==11.1.0 diff --git a/scripts/check-revocation.py b/scripts/check-revocation.py new file mode 100644 index 00000000..f0f157a4 --- /dev/null +++ b/scripts/check-revocation.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Regression check: logout must not be bypassed while Valkey is unavailable. + +Scenario (the one that used to fail): + 1. login and warm any session cache; + 2. pause Valkey; + 3. logout -> 204 (session row removed from PostgreSQL); + 4. resume Valkey; + 5. after the cache circuit breaker window, the old token must still be 401 on + every replica, and it must not be able to send a message. + +Requires the docker CLI and a running lab3 stack. +""" +import argparse +import os +import subprocess +import sys +import time +import uuid + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tests")) +from client import Client # noqa: E402 + + +def _docker(*args: str) -> None: + subprocess.run(["docker", *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + parser.add_argument("--valkey-container", default=os.environ.get("VALKEY_CONTAINER", "messenger-lab3-valkey-1")) + parser.add_argument("--wait", type=float, default=6.0, help="seconds to wait after resuming Valkey") + args = parser.parse_args() + + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + + # Warm the cache on both replicas. + for _ in range(6): + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + c.message(alice, chat, "warm-up before revocation") + + print("Pausing Valkey (%s) ..." % args.valkey_container) + _docker("pause", args.valkey_container) + try: + logout = c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + assert logout["status"] == 204, logout["status"] + print("Logout returned 204 while the cache was unavailable.") + finally: + _docker("unpause", args.valkey_container) + print("Valkey resumed; waiting %.1fs for the cache circuit breaker ..." % args.wait) + time.sleep(args.wait) + + seen = set() + for _ in range(12): + response = c.call("GET", "/api/v1/users/me", 401, actor=alice, schema="Error") + seen.add(response["headers"].get("X-Instance-Id")) + assert len(seen) >= 1, "no replica observed" + + blocked = c.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 401, + actor=alice, + body={"text": "must not be written"}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Error", + ) + assert blocked["status"] == 401, blocked["status"] + + print( + "PASS: revoked session stays 401 through %d replica(s) and cannot post after a " + "failed cache invalidation." % len(seen) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check-s3-outage.py b/scripts/check-s3-outage.py new file mode 100644 index 00000000..5c65aeaa --- /dev/null +++ b/scripts/check-s3-outage.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""S3 outage probe (grade 4): losing object storage must degrade, not crash. + + * stopping S3 -> /health/ready = 503 and upload/download = 503 (bounded); + * PostgreSQL data (users/chats/messages/metadata) stays intact; + * after S3 returns the API serves uploads again. + +Run from the repository root where ``docker compose`` is available. +""" +from __future__ import annotations + +import base64 +import json +import os +import struct +import subprocess +import time +import urllib.error +import urllib.request +import uuid +import zlib + +BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + + +def _png_chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + +def make_png(width=64, height=64): + pixels = os.urandom(width * height * 3) + raw = bytearray() + for row in range(height): + raw.append(0) + raw += pixels[row * width * 3 : (row + 1) * width * 3] + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _png_chunk(b"IHDR", ihdr) + _png_chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _png_chunk(b"IEND", b"") + + +def call(method, path, token=None, body=None, raw=None, content_type=None): + headers = {"Accept": "application/json"} + data = raw + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if content_type: + headers["Content-Type"] = content_type + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + except (urllib.error.URLError, TimeoutError, OSError): + return 0, b"" + + +def compose(*args): + subprocess.run(["docker", "compose", *args], check=True, stdout=subprocess.DEVNULL) + + +def wait_ready(expected, timeout=180.0): + deadline = time.time() + timeout + while time.time() < deadline: + if call("GET", "/health/ready")[0] == expected: + return + time.sleep(2) + raise SystemExit(f"readiness did not reach {expected}") + + +def main() -> None: + wait_ready(200) + username = "s3out_" + uuid.uuid4().hex[:8] + password = "T3st-" + uuid.uuid4().hex + assert call("POST", "/api/v1/users", body={"username": username, "display_name": "S3", "password": password})[0] == 201 + credential = base64.b64encode(f"{username}:{password}".encode()).decode() + request = urllib.request.Request( + BASE + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential, "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=15) as response: + token = json.loads(response.read())["token"] + user_id = json.loads(call("GET", "/api/v1/users/me", token=token)[1])["id"] + chat = json.loads(call("POST", "/api/v1/chats", token=token, body={"title": "s3", "member_ids": [user_id]})[1]) + assert call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", token=token, + raw=make_png(), content_type="image/png")[0] == 201 + print("baseline: upload = 201") + + print("stopping s3 ...") + compose("stop", "s3") + time.sleep(3) + ready = call("GET", "/health/ready")[0] + upload = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=b.png", token=token, + raw=make_png(), content_type="image/png")[0] + assert ready == 503, f"readiness must be 503 without S3, got {ready}" + assert upload == 503, f"upload must be a bounded 503 without S3, got {upload}" + print("s3 down: /health/ready = 503, upload = 503 (bounded)") + # PostgreSQL-backed endpoints still work. + assert call("GET", "/api/v1/users/me", token=token)[0] == 200 + print("s3 down: /users/me = 200 (metadata unaffected)") + + compose("start", "s3") + wait_ready(200, timeout=240) + again = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=c.png", token=token, + raw=make_png(), content_type="image/png")[0] + assert again == 201, f"upload must recover, got {again}" + print("s3 restored: upload = 201") + print("PASS: S3 outage degrades to bounded 503 and recovers") + + +if __name__ == "__main__": + main() diff --git a/scripts/check-upload-load.py b/scripts/check-upload-load.py new file mode 100644 index 00000000..ea9e8d68 --- /dev/null +++ b/scripts/check-upload-load.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Grade-4 load check: >=10 concurrent uploads, bounded API memory. + +Uploads N large valid PNGs in parallel through the balancer while sampling the +memory of the API replicas, then prints the measured peak. Files are generated +with the standard library only (a valid PNG with stored, not deflated, pixels). + +Run from the repository root (needs the docker CLI and a running lab4 stand): + + python3 scripts/check-upload-load.py --uploads 12 --megabytes 6 +""" +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import json +import os +import struct +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +import zlib + + +def _png_chunk(tag: bytes, data: bytes) -> bytes: + return ( + struct.pack(">I", len(data)) + + tag + + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + ) + + +def make_png(width: int, height: int) -> bytes: + pixels = os.urandom(width * height * 3) + raw = bytearray() + stride = width * 3 + for row in range(height): + raw.append(0) # filter type 0 + raw += pixels[row * stride : (row + 1) * stride] + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", ihdr) + + _png_chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + + _png_chunk(b"IEND", b"") + ) + + +def call(base, method, path, token=None, body=None, raw=None, content_type=None): + headers = {"Accept": "application/json"} + data = raw + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if content_type: + headers["Content-Type"] = content_type + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(base + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=60) as response: + payload = response.read() + return response.status, payload + except urllib.error.HTTPError as error: + return error.code, error.read() + + +def current_api_memory_mb() -> float: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.MemUsage}}"], + capture_output=True, text=True, + ).stdout + total = 0.0 + for line in out.splitlines(): + name, _, usage = line.partition("|") + if "api" not in name: + continue + value = usage.split("/")[0].strip() + number = "".join(ch for ch in value if ch.isdigit() or ch == ".") + if value.endswith("GiB"): + total += float(number) * 1024 + elif value.endswith("MiB"): + total += float(number) + elif value.endswith("KiB"): + total += float(number) / 1024 + return total + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + parser.add_argument("--uploads", type=int, default=12) + parser.add_argument("--megabytes", type=int, default=6) + args = parser.parse_args() + base = args.base_url.rstrip("/") + + username = "load_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + status, _ = call(base, "POST", "/api/v1/users", body={ + "username": username, "display_name": "Load", "password": password}) + assert status == 201, status + credential = base64.b64encode(f"{username}:{password}".encode()).decode() + request = urllib.request.Request( + base + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential, "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=15) as response: + token = json.loads(response.read())["token"] + status, user = call(base, "GET", "/api/v1/users/me", token=token) + user_id = json.loads(user)["id"] + status, chat_bytes = call(base, "POST", "/api/v1/chats", token=token, + body={"title": "load", "member_ids": [user_id]}) + assert status == 201, status + chat_id = json.loads(chat_bytes)["id"] + + side = int((args.megabytes * 1024 * 1024 / 3) ** 0.5) + payload = make_png(side, side) + print(json.dumps({"file_bytes": len(payload), "uploads": args.uploads})) + + peak = {"value": current_api_memory_mb()} + stop = threading.Event() + + def sampler(): + while not stop.is_set(): + peak["value"] = max(peak["value"], current_api_memory_mb()) + time.sleep(0.2) + + thread = threading.Thread(target=sampler, daemon=True) + thread.start() + started = time.time() + results = [] + + def upload(_): + return call(base, "POST", f"/api/v1/chats/{chat_id}/attachments?filename=load.png", + token=token, raw=payload, content_type="image/png") + + with concurrent.futures.ThreadPoolExecutor(max_workers=args.uploads) as pool: + for status_code, body in pool.map(upload, range(args.uploads)): + results.append(status_code) + + stop.set() + thread.join(timeout=2) + elapsed = time.time() - started + print(json.dumps({ + "ok": results.count(201), + "errors": [s for s in results if s != 201], + "seconds": round(elapsed, 2), + "api_peak_mem_mb": round(peak["value"], 1), + }, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh new file mode 100755 index 00000000..db47e0b9 --- /dev/null +++ b/scripts/gen-secrets.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu +# Generate local (gitignored) compose secrets with random credentials. +# Writes database, cache and S3 credentials plus the SeaweedFS S3 config that +# declares the limited "messenger-api" identity (separate from root). +cd "$(dirname "$0")/.." +mkdir -p secrets + +BUCKET="${S3_BUCKET:-messenger-attachments}" + +rand() { head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n'; } + +PASSWORD=$(rand) +ROOT_KEY="root$(rand)" +ROOT_SECRET=$(rand) +API_KEY="api$(rand)" +API_SECRET=$(rand) + +printf '%s' "$PASSWORD" > secrets/database_password +printf '%s' "postgresql://messenger:${PASSWORD}@postgres:5432/messenger" > secrets/database_url +printf '%s' "redis://valkey:6379/0" > secrets/cache_url + +printf '%s' "$ROOT_KEY" > secrets/s3_root_access_key_id +printf '%s' "$ROOT_SECRET" > secrets/s3_root_secret_access_key +printf '%s' "$API_KEY" > secrets/s3_access_key_id +printf '%s' "$API_SECRET" > secrets/s3_secret_access_key + +# The API identity may only Read/Write/List THIS bucket; root (Admin) is used +# solely by the one-shot s3-init bootstrap. +cat > secrets/s3_config.json </dev/null || true +echo "Wrote ./secrets/{database_password,database_url,cache_url,s3_*,s3_config.json} (not committed)." diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..94faae43 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +# Restart the API replicas, the balancer and ALL state stores used by this stack +# (PostgreSQL, Valkey, S3). Volumes are preserved; schema, bucket and data are +# never dropped or re-seeded. Used by: +# make test-persistence ACTION_SCRIPT=scripts/restart.sh +cd "$(dirname "$0")/.." +docker compose restart api lb postgres valkey s3 +exit 0 diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 00000000..42b6544e --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +# Restore the API service to the desired replica count after a failover test. +# Stopped replicas are started again; volumes and data are untouched. +cd "$(dirname "$0")/.." +docker compose up -d --no-recreate api +exit 0 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..9bbfc91b --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu +# Sequential (rolling) replacement of the API replicas under load. +# +# The image is rebuilt once, then each running replica is stopped (the balancer +# drains it and keeps serving from the peers) and recreated from the new image. +# Before the next replica is touched we wait until every running replica is +# healthy, so the balancer always has somewhere to send traffic. +cd "$(dirname "$0")/.." + +N="${API_REPLICAS:-2}" + +wait_healthy() { + for _ in $(seq 1 90); do + total=$(docker compose ps -q api | wc -l) + healthy=$(docker ps --filter label=com.docker.compose.service=api --filter health=healthy --format '{{.ID}}' | wc -l) + if [ "$total" -ge "$N" ] && [ "$healthy" -ge "$total" ]; then + return 0 + fi + sleep 1 + done +} + +echo "Building new API image..." +docker compose build api + +# Wait for the current generation to be healthy before starting. +wait_healthy + +for id in $(docker compose ps -q api); do + echo "Replacing replica $id ..." + docker stop -t 10 "$id" >/dev/null + docker rm "$id" >/dev/null + docker compose up -d --no-recreate api >/dev/null + wait_healthy +done + +echo "Rolling update complete; all replicas healthy." diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2547395a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +# Stop exactly ONE API replica and leave it stopped. +# +# TARGET_INSTANCE_ID is the X-Instance-Id observed by the test, which equals the +# replica's container hostname (Docker's short container id). We map it to the +# running api container and stop it with a short grace period so the hook +# returns well within the 10 second budget. PostgreSQL, Valkey and the balancer +# are never touched. +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." + +target="" +for id in $(docker ps --filter "label=com.docker.compose.service=api" --format '{{.ID}}'); do + case "$id" in + "$TARGET_INSTANCE_ID"*) target="$id" ;; + esac +done + +if [ -z "$target" ]; then + echo "no running API replica matches X-Instance-Id=$TARGET_INSTANCE_ID" >&2 + exit 1 +fi + +docker stop -t 5 "$target" >/dev/null +exit 0 diff --git a/secrets.example/README.md b/secrets.example/README.md new file mode 100644 index 00000000..760dd1b4 --- /dev/null +++ b/secrets.example/README.md @@ -0,0 +1,19 @@ +# Example only. Copy the whole directory to ./secrets/ (gitignored) and put a +# real local password in. Never commit real credentials. +# +# mkdir -p secrets +# cp secrets.example/* secrets/ +# # then edit the values below +# +# Or generate everything at once with random credentials: +# sh scripts/gen-secrets.sh +# +# Files expected by compose.yaml: +# database_password postgres password +# database_url postgresql://messenger:@postgres:5432/messenger +# cache_url redis://valkey:6379/0 +# s3_root_access_key_id SeaweedFS root access key (bootstrap only) +# s3_root_secret_access_key SeaweedFS root secret +# s3_access_key_id limited API access key +# s3_secret_access_key limited API secret +# s3_config.json SeaweedFS s3 config with both identities diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..19d86e09 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""MTUSI Messenger lab3 application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..a7944fe4 --- /dev/null +++ b/src/app.py @@ -0,0 +1,556 @@ +"""FastAPI application implementing the lab4 HTTP contract. + +Canonical contract: ``contracts/openapi.json`` and ``contracts/README.md``. +PostgreSQL is the source of truth for metadata; bytes live in a private S3 +bucket. Valkey is a best-effort session cache. Every response carries a +non-empty ``X-Request-Id`` and ``X-Instance-Id``. Messages are idempotent per +``Idempotency-Key`` and may reference attachments of the same chat. +""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import logging +import re +import tempfile +import uuid +from contextlib import asynccontextmanager +from typing import Dict, List, Optional + +from fastapi import Depends, FastAPI, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from psycopg import InterfaceError, OperationalError +from psycopg_pool import PoolTimeout +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import imaging, invalidation, storage +from . import repositories as repo +from . import security, sessions, state +from .config import settings +from .db import pool +from .errors import ApiError +from .instance import INSTANCE_ID +from .pagination import bound_cursor, encode_cursor + +logger = logging.getLogger("messenger") + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +_BEARER_RE = re.compile(r"^Bearer\s+(\S+)$", re.IGNORECASE) +_IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +_STATUS_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 405: "invalid_request", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", +} + +@asynccontextmanager +async def lifespan(_: FastAPI): + invalidation.start_worker() + try: + yield + finally: + invalidation.stop_worker() + + +app = FastAPI( + title="MTUSI Messenger — лабораторная 4", + version="1.4.0", + docs_url=None, + redoc_url=None, + openapi_url=None, + lifespan=lifespan, +) + + +# --------------------------------------------------------------------------- # +# Error envelope +# --------------------------------------------------------------------------- # +def _request_id(request: Request) -> str: + return getattr(request.state, "request_id", None) or uuid.uuid4().hex + + +def error_response( + request: Request, status: int, code: str, message: str, challenge: Optional[str] = None +) -> JSONResponse: + rid = _request_id(request) + headers = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID} + if challenge: + headers["WWW-Authenticate"] = challenge + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers=headers, + ) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = rid + try: + response = await call_next(request) + except Exception as exc: # noqa: BLE001 - never leak internals + logging.getLogger("messenger").error("unhandled error: %s", type(exc).__name__) + logging.getLogger("messenger").debug("traceback", exc_info=exc) + response = error_response(request, 500, "internal_error", "Внутренняя ошибка") + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + challenge = exc.challenge or (BEARER_CHALLENGE if exc.status == 401 else None) + return error_response(request, exc.status, exc.code, exc.message, challenge) + + +@app.exception_handler(RequestValidationError) +async def handle_validation(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse: + code = _STATUS_CODES.get(exc.status_code, "invalid_request") + challenge = BEARER_CHALLENGE if exc.status_code == 401 else None + return error_response(request, exc.status_code, code, "Ошибка запроса", challenge) + + +def _handle_db_error(request: Request, exc: Exception) -> JSONResponse: + logger.warning("database unavailable: %s", type(exc).__name__) + return error_response(request, 503, "unavailable", "База данных недоступна") + + +app.add_exception_handler(OperationalError, _handle_db_error) +app.add_exception_handler(InterfaceError, _handle_db_error) +app.add_exception_handler(PoolTimeout, _handle_db_error) + + +# --------------------------------------------------------------------------- # +# Request schemas +# --------------------------------------------------------------------------- # +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def _not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("display_name must not be blank") + return value + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: List[uuid.UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def _not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must not be blank") + return value + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: List[uuid.UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def _unique(cls, value: List[uuid.UUID]) -> List[uuid.UUID]: + if len(set(map(str, value))) != len(value): + raise ValueError("duplicate attachment_ids") + return value + + @model_validator(mode="after") + def _non_empty(self) -> "CreateMessage": + if not self.text.strip() and not self.attachment_ids: + raise ValueError("message must have text or attachments") + return self + + +# --------------------------------------------------------------------------- # +# Dependencies and helpers +# --------------------------------------------------------------------------- # +def get_conn(): + with pool().connection() as conn: + yield conn + + +def _canonical_uuid(value: str) -> str: + try: + return str(uuid.UUID(value)) + except (ValueError, AttributeError, TypeError): + raise ApiError(400, "invalid_request", "Некорректный идентификатор") + + +def _parse_basic(request: Request) -> tuple[str, str]: + header = request.headers.get("Authorization", "").strip() + if not header.lower().startswith("basic "): + raise ApiError(401, "unauthorized", "Требуется Basic-аутентификация", challenge=BASIC_CHALLENGE) + encoded = header[6:].strip() + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise ApiError(401, "unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + if ":" not in decoded: + raise ApiError(401, "unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + username, password = decoded.split(":", 1) + return username, password + + +def _bearer_token(request: Request) -> str: + header = request.headers.get("Authorization", "").strip() + match = _BEARER_RE.match(header) + if not match: + raise ApiError(401, "unauthorized", "Требуется Bearer-токен", challenge=BEARER_CHALLENGE) + return match.group(1) + + +def _idempotency_key(request: Request) -> str: + value = (request.headers.get("Idempotency-Key") or "").strip() + if not value or not _IDEMPOTENCY_RE.match(value): + raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key") + return value + + +def current_user(request: Request, conn=Depends(get_conn)) -> Dict: + token = _bearer_token(request) + user_id = sessions.resolve(conn, token) + if user_id is None: + raise ApiError(401, "unauthorized", "Сессия недействительна", challenge=BEARER_CHALLENGE) + row = repo.get_user_by_id(conn, user_id) + if row is None: + raise ApiError(401, "unauthorized", "Сессия недействительна", challenge=BEARER_CHALLENGE) + return row + + +def _require_chat(conn, chat_id: str, user_id: str) -> str: + canonical = _canonical_uuid(chat_id) + if repo.get_chat_for(conn, canonical, user_id) is None: + raise ApiError(404, "not_found", "Чат недоступен") + return canonical + + +def _require_chat_standalone(chat_id: str, user_id: str) -> str: + """Chat-membership check on its own connection (used by the async upload).""" + with pool().connection() as conn: + return _require_chat(conn, chat_id, user_id) + + +def _validate_filename(name: str) -> str: + if not name or len(name) > 255: + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + if any(ch in name for ch in ("/", "\\", "\x00")) or any( + ord(ch) < 0x20 or ord(ch) == 0x7F for ch in name + ): + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + return name + + +def _validate_attachments(conn, chat_id: str, attachment_ids: List[str]) -> List[str]: + """All referenced attachments must exist in THIS chat and be ready.""" + if not attachment_ids: + return [] + rows = repo.attachments_in_chat(conn, chat_id, attachment_ids) + found = {str(row["id"]): row["status"] for row in rows} + if len(found) != len(set(attachment_ids)): + raise ApiError(404, "not_found", "Вложение недоступно") + for attachment_id in attachment_ids: + if found.get(attachment_id) != "ready": + raise ApiError(409, "conflict", "Вложение не готово") + return attachment_ids + + +def _finalize_upload( + chat_id: str, owner_id: str, filename: str, fileobj, size_bytes: int, sha256: str +) -> Dict: + """Validate the image, store the bytes in S3 and commit metadata. + + Runs in a worker thread. The bytes are validated from the temporary file + (never trusted from headers) and uploaded with a server-generated key. If + the metadata COMMIT fails after the PUT, the object is deleted so a failed + upload does not leave an untracked object behind (best-effort; the + reconciler covers crash windows, see scripts/reconcile-s3.py). + """ + content_type, _width, _height = imaging.inspect(fileobj) + attachment_id = str(uuid.uuid4()) + key = f"chats/{chat_id}/{attachment_id}" + fileobj.seek(0) + storage.put_object(key, fileobj, content_type) + try: + with pool().connection() as conn: + now = repo.utcnow() + row = repo.insert_attachment( + conn, + attachment_id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + sha256, + key, + "ready", + None, + now, + now, + ) + return repo.attachment_json(row) + except Exception: + if storage.delete_object(key): + logger.warning("compensated orphan S3 object %s", key) + raise + + +# --------------------------------------------------------------------------- # +# Health +# --------------------------------------------------------------------------- # +@app.get("/health/live") +def liveness() -> dict: + return {"status": "ok"} + + +@app.get("/health/ready") +def readiness() -> dict: + if state.is_shutting_down(): + raise ApiError(503, "unavailable", "Реплика останавливается") + try: + with pool().connection(timeout=2) as conn: + conn.execute("select 1") + except (OperationalError, InterfaceError, PoolTimeout): + raise ApiError(503, "unavailable", "База данных недоступна") + if not storage.ready(): + raise ApiError(503, "unavailable", "Хранилище недоступно") + return {"status": "ok"} + + +# --------------------------------------------------------------------------- # +# Users and sessions +# --------------------------------------------------------------------------- # +@app.post("/api/v1/users", status_code=201) +def register_user(payload: CreateUser, conn=Depends(get_conn)) -> dict: + password_hash = security.hash_password(payload.password) + row = repo.create_user(conn, payload.username, payload.display_name, password_hash) + return repo.user_json(row) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request, conn=Depends(get_conn)) -> dict: + username, password = _parse_basic(request) + row = repo.get_user_with_hash_by_username(conn, username) + if row is None or not security.verify_password(row["password_hash"], password): + raise ApiError(401, "unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + token, expires_at = sessions.create(conn, str(row["id"])) + return { + "token": token, + "token_type": "Bearer", + "expires_at": repo.rfc3339(expires_at), + "user": repo.user_json(row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session( + request: Request, + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> Response: + sessions.revoke(conn, _bearer_token(request)) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def current_user_endpoint(user: Dict = Depends(current_user)) -> dict: + return repo.user_json(user) + + +# --------------------------------------------------------------------------- # +# Chats +# --------------------------------------------------------------------------- # +@app.post("/api/v1/chats", status_code=201) +def create_chat(payload: CreateChat, user: Dict = Depends(current_user), conn=Depends(get_conn)) -> dict: + members = [str(member) for member in payload.member_ids] + if len(set(members)) != len(members): + raise ApiError(400, "invalid_request", "Дубликаты участников") + if str(user["id"]) not in members: + raise ApiError(400, "invalid_request", "Вызывающий должен входить в участников") + if repo.count_users(conn, members) != len(set(members)): + raise ApiError(400, "invalid_request", "Неизвестный участник") + row = repo.create_chat(conn, payload.title, members) + return repo.chat_json(row) + + +@app.get("/api/v1/chats") +def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> dict: + key = bound_cursor(cursor, "chats") + rows = repo.list_chats(conn, str(user["id"]), limit + 1, key) + page = rows[:limit] + next_cursor = None + if len(rows) > limit: + next_cursor = encode_cursor("chats", page[-1]["created_at"], str(page[-1]["id"])) + return {"items": [repo.chat_json(row) for row in page], "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, user: Dict = Depends(current_user), conn=Depends(get_conn)) -> dict: + canonical = _require_chat(conn, chat_id, str(user["id"])) + row = repo.get_chat_for(conn, canonical, str(user["id"])) + return repo.chat_json(row) + + +# --------------------------------------------------------------------------- # +# Messages +# --------------------------------------------------------------------------- # +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + payload: CreateMessage, + request: Request, + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> dict: + canonical = _require_chat(conn, chat_id, str(user["id"])) + key = _idempotency_key(request) + attachment_ids = _validate_attachments( + conn, canonical, [str(item) for item in payload.attachment_ids] + ) + scope = f"POST /api/v1/chats/{canonical}/messages" + return repo.create_message_idempotent( + conn, + canonical, + str(user["id"]), + payload.text, + attachment_ids, + scope, + key, + settings.idempotency_ttl_seconds, + ) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> dict: + canonical = _require_chat(conn, chat_id, str(user["id"])) + key = bound_cursor(cursor, canonical) + rows = repo.list_messages(conn, canonical, limit + 1, key) + page = rows[:limit] + next_cursor = None + if len(rows) > limit: + next_cursor = encode_cursor(canonical, page[-1]["created_at"], str(page[-1]["id"])) + return {"items": [repo.message_json(row) for row in page], "next_cursor": next_cursor} + + +# --------------------------------------------------------------------------- # +# Attachments (lab4) +# --------------------------------------------------------------------------- # +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + user: Dict = Depends(current_user), +) -> dict: + canonical = await run_in_threadpool(_require_chat_standalone, chat_id, str(user["id"])) + safe_name = _validate_filename(filename) + + content_length = request.headers.get("Content-Length", "") + if content_length.isdigit() and int(content_length) > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + + # Stream the raw bytes to a bounded spill-to-disk file, hashing as we go, so + # a single upload never holds the whole body in RAM. + spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024) + size = 0 + digest = hashlib.sha256() + try: + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + digest.update(chunk) + spool.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "Пустой файл") + spool.seek(0) + return await run_in_threadpool( + _finalize_upload, + canonical, + str(user["id"]), + safe_name, + spool, + size, + digest.hexdigest(), + ) + finally: + spool.close() + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment( + attachment_id: str, user: Dict = Depends(current_user), conn=Depends(get_conn) +) -> dict: + canonical = _canonical_uuid(attachment_id) + row = repo.get_attachment_for(conn, canonical, str(user["id"])) + if row is None: + raise ApiError(404, "not_found", "Вложение недоступно") + return repo.attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + variant: str = Query(default="original"), + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> StreamingResponse: + canonical = _canonical_uuid(attachment_id) + row = repo.get_attachment_for(conn, canonical, str(user["id"])) + if row is None: + raise ApiError(404, "not_found", "Вложение недоступно") + if variant != "original" or row["status"] != "ready": + raise ApiError(404, "not_found", "Вариант не найден") + obj = storage.open_object(row["s3_key"]) + body = obj["Body"] + + def stream(): + try: + for chunk in body.iter_chunks(chunk_size=64 * 1024): + yield chunk + finally: + body.close() + + return StreamingResponse(stream(), media_type=row["content_type"]) diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..abc0895e --- /dev/null +++ b/src/config.py @@ -0,0 +1,83 @@ +"""Runtime configuration from environment variables / secret files. + +Only logical interface names from ``.env.example`` are consumed; secret values +are read from files (``*_FILE``) so that no credential is baked into the image +or committed. The mapping is documented in REPORT.md. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _read_secret(file_key: str, value_key: str, default: str = "") -> str: + path = os.environ.get(file_key) + if path: + try: + return Path(path).read_text(encoding="utf-8").strip() + except OSError: + return default + return os.environ.get(value_key, default) + + +@dataclass(frozen=True) +class Settings: + app_host: str + app_port: int + app_env: str + session_ttl_seconds: int + session_store: str + session_cache_ttl_seconds: int + cache_invalidation_interval_seconds: float + idempotency_ttl_seconds: int + database_url: str + cache_url: str + db_pool_min: int + db_pool_max: int + db_pool_timeout: float + db_statement_timeout_ms: int + # S3 / attachment limits + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + + @property + def cache_enabled(self) -> bool: + return bool(self.cache_url) and self.session_store in {"valkey", "redis"} + + +def load_settings() -> Settings: + return Settings( + app_host=os.environ.get("APP_HOST", "0.0.0.0"), + app_port=int(os.environ.get("APP_PORT", "8080")), + app_env=os.environ.get("APP_ENV", "lab"), + session_ttl_seconds=int(os.environ.get("SESSION_TTL_SECONDS", "3600")), + session_store=os.environ.get("SESSION_STORE", "postgres").lower(), + session_cache_ttl_seconds=int(os.environ.get("SESSION_CACHE_TTL_SECONDS", "60")), + cache_invalidation_interval_seconds=float( + os.environ.get("CACHE_INVALIDATION_INTERVAL_SECONDS", "0.5") + ), + idempotency_ttl_seconds=int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")), + database_url=_read_secret("DATABASE_URL_FILE", "DATABASE_URL"), + cache_url=_read_secret("CACHE_URL_FILE", "CACHE_URL"), + db_pool_min=int(os.environ.get("DB_POOL_MIN", "1")), + db_pool_max=int(os.environ.get("DB_POOL_MAX", "10")), + db_pool_timeout=float(os.environ.get("DB_POOL_TIMEOUT", "5")), + db_statement_timeout_ms=int(os.environ.get("DB_STATEMENT_TIMEOUT_MS", "5000")), + s3_endpoint=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + s3_region=os.environ.get("S3_REGION", "us-east-1"), + s3_bucket=os.environ.get("S3_BUCKET", "messenger-attachments"), + s3_access_key_id=_read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), + s3_secret_access_key=_read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), + # 10 MiB per contract; allow one extra byte to detect "too large". + max_upload_bytes=int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024))), + max_image_pixels=int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")), + ) + + +settings = load_settings() diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..5353efc5 --- /dev/null +++ b/src/db.py @@ -0,0 +1,42 @@ +"""PostgreSQL access: a bounded connection pool with timeouts. + +All queries elsewhere are parameterized; this module only owns the pool. +A statement timeout and a bounded pool are part of the grade-4 requirements. +""" +from __future__ import annotations + +import logging +from typing import Optional + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import settings + +logger = logging.getLogger("messenger.db") + +_pool: Optional[ConnectionPool] = None + + +def pool() -> ConnectionPool: + global _pool + if _pool is None: + _pool = ConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_pool_timeout, + kwargs={ + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.db_statement_timeout_ms}", + }, + open=True, + ) + return _pool + + +def close_pool() -> None: + global _pool + if _pool is not None: + _pool.close() + _pool = None diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..48f9a0f0 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,20 @@ +"""Shared error type mapped to the contract error envelope.""" +from __future__ import annotations + +from typing import Optional + + +class ApiError(Exception): + def __init__( + self, + status: int, + code: str, + message: str, + challenge: Optional[str] = None, + ) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + # Optional WWW-Authenticate value (used by 401 responses). + self.challenge = challenge diff --git a/src/imaging.py b/src/imaging.py new file mode 100644 index 00000000..0279ec50 --- /dev/null +++ b/src/imaging.py @@ -0,0 +1,50 @@ +"""Image validation for attachments. + +The real format is decided by the bytes (Pillow), never by the client's +``Content-Type`` or by ``filename``. Pixel count is checked from the header +*before* any full decode, so a decompression bomb is rejected cheaply. +""" +from __future__ import annotations + +import logging + +from PIL import Image, UnidentifiedImageError + +from .config import settings +from .errors import ApiError + +logger = logging.getLogger("messenger.imaging") + +# Pillow refuses images above this many pixels outright. +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + +_CONTENT_TYPES = {"PNG": "image/png", "JPEG": "image/jpeg"} + + +def inspect(fileobj) -> tuple[str, int, int]: + """Return ``(content_type, width, height)`` or raise ``ApiError``. + + 415 for anything that is not a valid PNG/JPEG, 413 for too many pixels. + """ + fileobj.seek(0) + try: + image = Image.open(fileobj) # header only; no full decode yet + fmt = image.format + width, height = image.size + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError): + raise ApiError(415, "unsupported_media_type", "Поддерживаются только PNG и JPEG") + + content_type = _CONTENT_TYPES.get(fmt) + if content_type is None: + raise ApiError(415, "unsupported_media_type", "Поддерживаются только PNG и JPEG") + + if width * height > settings.max_image_pixels: + raise ApiError(413, "payload_too_large", "Слишком большое изображение") + + # Validate the container without decoding every pixel into memory. + try: + image.verify() + except Exception: # noqa: BLE001 - any Pillow failure is an invalid image + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") + + return content_type, width, height diff --git a/src/instance.py b/src/instance.py new file mode 100644 index 00000000..8411c30e --- /dev/null +++ b/src/instance.py @@ -0,0 +1,21 @@ +"""Stable identity of this API process, exposed as ``X-Instance-Id``. + +The value is fixed for the whole life of the process, so clients and the +balancer can tell replicas apart. Inside Docker the default container hostname +already is the (short) container id, which makes the value easy to map back to +a real container without exposing a routable address. +""" +from __future__ import annotations + +import os +import socket + + +def _resolve() -> str: + value = os.environ.get("INSTANCE_ID", "").strip() + if value: + return value + return socket.gethostname() or "api" + + +INSTANCE_ID = _resolve() diff --git a/src/invalidation.py b/src/invalidation.py new file mode 100644 index 00000000..aa534ac1 --- /dev/null +++ b/src/invalidation.py @@ -0,0 +1,76 @@ +"""Durable cache-invalidation drainer (lab3). + +Logout writes a pending invalidation to PostgreSQL in the same transaction as +the authoritative session delete. This tiny background thread (one per API +replica, not an external broker) removes the corresponding Valkey key and then +the queue row. Multiple replicas may process the same row; cache deletes are +idempotent, so this is safe and needs no distributed lock. + +The worker is deliberately not on the request path: it only runs when a cache +is configured, and it retries forever while Valkey is unreachable. That is what +lets a logout survive a Valkey outage: the session is already gone from +PostgreSQL, and the stale cache entry is evicted as soon as the cache returns. +""" +from __future__ import annotations + +import logging +import threading +from typing import Optional + +from . import repositories as repo +from . import sessions +from .config import settings +from .db import pool + +logger = logging.getLogger("messenger.invalidation") + +_BATCH = 200 +_stop = threading.Event() +_thread: Optional[threading.Thread] = None + + +def _drain_once() -> None: + with pool().connection() as conn: + pending = repo.list_pending_invalidations(conn, _BATCH) + if not pending: + return + + invalidated = [token_hash for token_hash in pending if sessions.invalidate_session_key(token_hash)] + if not invalidated: + # Cache still unreachable; keep the rows for the next attempt. + return + with pool().connection() as conn: + repo.delete_cache_invalidations(conn, invalidated) + logger.debug("drained %d cache invalidations", len(invalidated)) + + +def _loop() -> None: + while not _stop.is_set(): + try: + _drain_once() + except Exception as exc: # noqa: BLE001 - never kill the drainer + logger.warning("cache invalidation drain failed: %s", type(exc).__name__) + _stop.wait(settings.cache_invalidation_interval_seconds) + + +def start_worker() -> None: + global _thread + if not settings.cache_enabled: + return + if _thread is not None and _thread.is_alive(): + return + _stop.clear() + _thread = threading.Thread(target=_loop, name="cache-invalidation", daemon=True) + _thread.start() + logger.info( + "cache invalidation drainer started (interval=%.2fs)", + settings.cache_invalidation_interval_seconds, + ) + + +def stop_worker() -> None: + global _thread + _stop.set() + if _thread is not None: + _thread.join(timeout=2) + _thread = None diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..502c57ff --- /dev/null +++ b/src/main.py @@ -0,0 +1,60 @@ +"""Container entrypoint: run the HTTP server as PID 1 with graceful shutdown. + +Migrations are executed by the dedicated one-shot ``migrate`` compose service +(``python -m src.migrate``) before any replica starts, so replicas do not race +on the schema. + +On SIGTERM/SIGINT the handler marks the process as shutting down: readiness +degrades to 503 so the balancer drains this replica, then Uvicorn stops +accepting connections and waits up to ``timeout_graceful_shutdown`` for +in-flight requests to finish. +""" +from __future__ import annotations + +import logging +import os + +import uvicorn + +from . import state +from .config import settings +from .instance import INSTANCE_ID + + +def main() -> None: + logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + logging.getLogger("messenger").info( + "starting messenger api instance=%s env=%s port=%s sessions=%s cache=%s", + INSTANCE_ID, + settings.app_env, + settings.app_port, + settings.session_store, + "on" if settings.cache_enabled else "off", + ) + + config = uvicorn.Config( + "src.app:app", + host=settings.app_host, + port=settings.app_port, + log_level=os.environ.get("LOG_LEVEL", "info").lower(), + access_log=True, + timeout_graceful_shutdown=10, + ) + server = uvicorn.Server(config) + + def handle_exit(sig, frame=None): # noqa: ANN001 - signal handler + state.begin_shutdown() + logging.getLogger("messenger").info("received signal %s; draining", sig) + server.should_exit = True + + # Uvicorn installs this exact method as its signal handler; overriding the + # bound attribute keeps cross-platform behaviour while adding our flag. + server.handle_exit = handle_exit + server.run() + + +if __name__ == "__main__": + main() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..76f17bb9 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,67 @@ +"""Versioned, idempotent migration runner. + +Applied versions are recorded in ``schema_migrations``; a transaction-scoped +advisory lock serializes concurrent callers. In lab3 migrations are executed by +a dedicated one-shot ``migrate`` service, not by every API replica, so schema +changes happen exactly once before any replica starts serving traffic. +""" +from __future__ import annotations + +import logging +import time +from pathlib import Path + +import psycopg +from psycopg_pool import PoolTimeout + +from .db import pool + +logger = logging.getLogger("messenger.migrate") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +LOCK_KEY = 776_2026 + + +def _apply_once() -> None: + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + with pool().connection() as conn: + conn.execute("select pg_advisory_xact_lock(%s)", (LOCK_KEY,)) + conn.execute( + """ + create table if not exists schema_migrations ( + version text primary key, + applied_at timestamptz not null default now() + ) + """ + ) + applied = { + row["version"] + for row in conn.execute("select version from schema_migrations").fetchall() + } + for path in files: + if path.name in applied: + continue + logger.info("applying migration %s", path.name) + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "insert into schema_migrations (version) values (%s)", (path.name,) + ) + + +def apply_migrations(attempts: int = 30, delay: float = 1.0) -> None: + last: Exception | None = None + for attempt in range(1, attempts + 1): + try: + _apply_once() + logger.info("migrations up to date") + return + except (psycopg.OperationalError, PoolTimeout) as exc: + last = exc + logger.warning("database not ready (attempt %s/%s): %s", attempt, attempts, exc) + time.sleep(delay) + raise RuntimeError(f"database unavailable after {attempts} attempts: {last}") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + apply_migrations() diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..c832d2ff --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,53 @@ +"""Opaque keyset cursors for ``(created_at, id)`` ascending pagination.""" +from __future__ import annotations + +import base64 +import binascii +import datetime as dt +import json +import uuid +from typing import Optional, Tuple + +from .errors import ApiError + +CursorKey = Tuple[dt.datetime, str] + + +def encode_cursor(resource_id: str, created_at: dt.datetime, item_id: str) -> str: + payload = json.dumps( + { + "r": resource_id, + "t": created_at.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "i": item_id, + }, + separators=(",", ":"), + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: str) -> Tuple[str, CursorKey]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(padded).decode("utf-8")) + if not isinstance(data, dict): + raise ValueError("cursor payload must be an object") + resource = data["r"] + item_id = data["i"] + if not isinstance(resource, str) or not isinstance(item_id, str): + raise ValueError("cursor fields must be strings") + created = dt.datetime.strptime(data["t"], "%Y-%m-%dT%H:%M:%S.%fZ").replace( + tzinfo=dt.timezone.utc + ) + return resource, (created, str(uuid.UUID(item_id))) + except (binascii.Error, ValueError, KeyError, TypeError, AttributeError, UnicodeDecodeError): + raise ApiError(400, "invalid_request", "Некорректный cursor") + + +def bound_cursor(cursor: str, expected_resource: str) -> Optional[CursorKey]: + """Validate resource binding and return the key, or None for empty cursor.""" + if cursor is None: + return None + resource, key = decode_cursor(cursor) + if resource != expected_resource: + raise ApiError(400, "invalid_request", "cursor относится к другому ресурсу") + return key diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..fbae67ed --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,75 @@ +"""Reconcile S3 objects and attachment metadata (lab4 grade-4 safety net). + +The synchronous upload spends a short window between the S3 PUT and the +PostgreSQL COMMIT. A crash there can leave an *orphan* object (in S3, no row); +the inline compensation in the API covers the ordinary failure path, and this +tool covers crash windows and any object created but never referenced. + + docker compose exec -T api python -m src.reconcile # dry-run + docker compose exec -T api python -m src.reconcile --apply # delete orphans + +Safety: + * dry-run by default; ``--apply`` is required to delete anything; + * only objects with no metadata row are candidates (referenced objects are + never touched); + * only objects older than ``--grace-seconds`` (default 3600) are deleted, so a + just-started upload is never mistaken for an orphan; + * objects present in metadata but missing in S3 are reported, never guessed. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json + +from . import repositories as repo +from . import storage +from .db import pool + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reconcile S3 objects and attachment metadata") + parser.add_argument("--apply", action="store_true", help="actually delete orphan objects") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--limit", type=int, default=50, help="how many orphans to print") + args = parser.parse_args() + + with pool().connection() as conn: + rows = repo.list_all_attachments(conn) + known = {str(row["s3_key"]): str(row["id"]) for row in rows} + + cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.grace_seconds) + objects = list(storage.list_keys("")) + s3_keys = {item["Key"]: item.get("LastModified") for item in objects} + + missing = sorted(key for key in known if key not in s3_keys) + orphans = [] + for key, modified in s3_keys.items(): + if key in known: + continue + if modified is not None and modified > cutoff: + continue # too fresh: may be an in-flight upload + orphans.append(key) + + report = { + "db_attachments": len(known), + "s3_objects": len(s3_keys), + "orphans": len(orphans), + "missing": len(missing), + "apply": bool(args.apply), + "grace_seconds": args.grace_seconds, + } + print(json.dumps(report, ensure_ascii=False)) + + for key in sorted(orphans)[: args.limit]: + print("orphan:", key) + for key in missing[: args.limit]: + print("missing:", key, "attachment_id=" + known[key]) + + if args.apply and orphans: + deleted = sum(1 for key in orphans if storage.delete_object(key)) + print(json.dumps({"deleted_orphans": deleted})) + + +if __name__ == "__main__": + main() diff --git a/src/repositories.py b/src/repositories.py new file mode 100644 index 00000000..34beaa08 --- /dev/null +++ b/src/repositories.py @@ -0,0 +1,509 @@ +"""Parameterized SQL for users, chats, members, messages, sessions and the +shared idempotency ledger. + +Every statement uses bound parameters (no string concatenation). Composite +writes happen inside one transaction so the caller gets all-or-nothing. +""" +from __future__ import annotations + +import datetime as dt +import hashlib +import json +import uuid +from typing import Dict, List, Optional, Sequence + +from psycopg.errors import UniqueViolation +from psycopg.types.json import Jsonb + +from .errors import ApiError +from .pagination import CursorKey + + +def utcnow() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def rfc3339(value: dt.datetime) -> str: + return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" + + +def _members(value) -> List[str]: + if value is None: + return [] + return [str(item) for item in value] + + +def user_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": rfc3339(row["created_at"]), + } + + +def chat_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": _members(row["member_ids"]), + "created_at": rfc3339(row["created_at"]), + } + + +def message_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": rfc3339(row["created_at"]), + "attachment_ids": _members(row.get("attachment_ids")), + } + + +_MESSAGE_ATTACHMENTS_SUBQUERY = ( + "(select array_agg(ma.attachment_id order by ma.position) " + "from message_attachments ma where ma.message_id = m.id)" +) + + +def attachment_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": rfc3339(row["created_at"]), + "updated_at": rfc3339(row["updated_at"]), + } + + +# --------------------------------------------------------------------------- # +# Users +# --------------------------------------------------------------------------- # +def create_user(conn, username: str, display_name: str, password_hash: str) -> Dict: + try: + row = conn.execute( + """ + insert into users (id, username, display_name, password_hash, created_at) + values (%s, %s, %s, %s, %s) + returning id, username, display_name, created_at + """, + (str(uuid.uuid4()), username, display_name, password_hash, utcnow()), + ).fetchone() + except UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя уже занято") + return row + + +def get_user_by_id(conn, user_id: str) -> Optional[Dict]: + return conn.execute( + "select id, username, display_name, created_at from users where id = %s", + (user_id,), + ).fetchone() + + +def get_user_with_hash_by_username(conn, username: str) -> Optional[Dict]: + return conn.execute( + "select id, username, display_name, created_at, password_hash from users where username = %s", + (username,), + ).fetchone() + + +def count_users(conn, user_ids: Sequence[str]) -> int: + row = conn.execute( + "select count(*) as n from users where id = any(%s::uuid[])", (list(user_ids),) + ).fetchone() + return int(row["n"]) + + +# --------------------------------------------------------------------------- # +# Chats +# --------------------------------------------------------------------------- # +def create_chat(conn, title: str, member_ids: Sequence[str]) -> Dict: + chat_id = str(uuid.uuid4()) + created = utcnow() + conn.execute( + "insert into chats (id, title, created_at) values (%s, %s, %s)", + (chat_id, title, created), + ) + with conn.cursor() as cur: + cur.executemany( + "insert into chat_members (chat_id, user_id) values (%s, %s)", + [(chat_id, member) for member in member_ids], + ) + return { + "id": chat_id, + "title": title, + "member_ids": list(member_ids), + "created_at": created, + } + + +_MEMBER_SUBQUERY = ( + "(select array_agg(m.user_id) from chat_members m where m.chat_id = c.id)" +) + + +def get_chat_for(conn, chat_id: str, user_id: str) -> Optional[Dict]: + return conn.execute( + f""" + select c.id, c.title, c.created_at, {_MEMBER_SUBQUERY} as member_ids + from chats c + where c.id = %s + and exists ( + select 1 from chat_members cm + where cm.chat_id = c.id and cm.user_id = %s + ) + """, + (chat_id, user_id), + ).fetchone() + + +def list_chats(conn, user_id: str, limit: int, cursor: Optional[CursorKey]) -> List[Dict]: + if cursor is None: + return conn.execute( + f""" + select c.id, c.title, c.created_at, {_MEMBER_SUBQUERY} as member_ids + from chats c + join chat_members cm on cm.chat_id = c.id + where cm.user_id = %s + order by c.created_at, c.id + limit %s + """, + (user_id, limit), + ).fetchall() + return conn.execute( + f""" + select c.id, c.title, c.created_at, {_MEMBER_SUBQUERY} as member_ids + from chats c + join chat_members cm on cm.chat_id = c.id + where cm.user_id = %s and (c.created_at, c.id) > (%s, %s) + order by c.created_at, c.id + limit %s + """, + (user_id, cursor[0], cursor[1], limit), + ).fetchall() + + +# --------------------------------------------------------------------------- # +# Messages +# --------------------------------------------------------------------------- # +def insert_message( + conn, chat_id: str, sender_id: str, text: str, attachment_ids: Sequence[str] = () +) -> Dict: + message_id = str(uuid.uuid4()) + created = utcnow() + conn.execute( + "insert into messages (id, chat_id, sender_id, text, created_at) values (%s, %s, %s, %s, %s)", + (message_id, chat_id, sender_id, text, created), + ) + attachments = list(attachment_ids) + if attachments: + with conn.cursor() as cur: + cur.executemany( + "insert into message_attachments (message_id, attachment_id, position) values (%s, %s, %s)", + [(message_id, attachment_id, position) for position, attachment_id in enumerate(attachments)], + ) + return { + "id": message_id, + "chat_id": chat_id, + "sender_id": sender_id, + "text": text, + "created_at": created, + "attachment_ids": attachments, + } + + +def list_messages( + conn, chat_id: str, limit: int, cursor: Optional[CursorKey] +) -> List[Dict]: + if cursor is None: + return conn.execute( + f""" + select m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {_MESSAGE_ATTACHMENTS_SUBQUERY} as attachment_ids + from messages m + where m.chat_id = %s + order by m.created_at, m.id + limit %s + """, + (chat_id, limit), + ).fetchall() + return conn.execute( + f""" + select m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {_MESSAGE_ATTACHMENTS_SUBQUERY} as attachment_ids + from messages m + where m.chat_id = %s and (m.created_at, m.id) > (%s, %s) + order by m.created_at, m.id + limit %s + """, + (chat_id, cursor[0], cursor[1], limit), + ).fetchall() + + +# --------------------------------------------------------------------------- # +# Shared idempotency ledger (lab3) +# --------------------------------------------------------------------------- # +def message_request_hash(text: str, attachment_ids: Sequence[str] = ()) -> str: + """Canonical hash of the semantic body. + + JSON key order and an absent ``attachment_ids`` versus ``[]`` must be + equivalent, and the order of attachment ids is not significant. The + canonical form is deterministic so any replica computes the same value. + """ + canonical = json.dumps( + {"text": text, "attachment_ids": sorted(attachment_ids or [])}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def get_idempotency(conn, user_id: str, scope: str, key: str) -> Optional[Dict]: + return conn.execute( + """ + select request_hash, status_code, response_body + from idempotency_keys + where user_id = %s and scope = %s and key = %s + """, + (user_id, scope, key), + ).fetchone() + + +def insert_idempotency( + conn, + user_id: str, + scope: str, + key: str, + request_hash: str, + status_code: int, + response_body: Dict, + ttl_seconds: int, +) -> None: + conn.execute( + """ + insert into idempotency_keys + (user_id, scope, key, request_hash, status_code, response_body, created_at, expires_at) + values (%s, %s, %s, %s, %s, %s, now(), now() + make_interval(secs => %s)) + """, + (user_id, scope, key, request_hash, status_code, Jsonb(response_body), ttl_seconds), + ) + + +def purge_expired_idempotency(conn) -> int: + rows = conn.execute( + "delete from idempotency_keys where expires_at <= now() returning key" + ).fetchall() + return len(rows) + + +def create_message_idempotent( + conn, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: Sequence[str], + scope: str, + key: str, + ttl_seconds: int, +) -> Dict: + """Create a message exactly once for (sender, scope, key). + + A transaction-scoped advisory lock keyed on the logical idempotency scope + serializes concurrent retries across *all* API replicas through PostgreSQL. + The first caller inserts the message (and its attachment links) and the + ledger row in one transaction; later callers observe the committed row and + replay the stored response. A reused key with different content conflicts. + """ + request_hash = message_request_hash(text, attachment_ids) + conn.execute( + "select pg_advisory_xact_lock(hashtext(%s), hashtext(%s))", + (sender_id + "|" + scope, key), + ) + existing = get_idempotency(conn, sender_id, scope, key) + if existing is not None: + if existing["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим сообщением") + return existing["response_body"] + message = insert_message(conn, chat_id, sender_id, text, attachment_ids) + body = message_json(message) + insert_idempotency(conn, sender_id, scope, key, request_hash, 201, body, ttl_seconds) + return body + + +# --------------------------------------------------------------------------- # +# Sessions (PostgreSQL is the source of truth) +# --------------------------------------------------------------------------- # +def insert_session(conn, token_hash: str, user_id: str, expires_at: dt.datetime) -> None: + conn.execute( + """ + insert into sessions (token_hash, user_id, created_at, expires_at) + values (%s, %s, %s, %s) + """, + (token_hash, user_id, utcnow(), expires_at), + ) + + +def get_valid_session(conn, token_hash: str) -> Optional[Dict]: + return conn.execute( + """ + select token_hash, user_id, expires_at + from sessions + where token_hash = %s and expires_at > now() + """, + (token_hash,), + ).fetchone() + + +def delete_session(conn, token_hash: str) -> None: + conn.execute("delete from sessions where token_hash = %s", (token_hash,)) + + +def purge_expired_sessions(conn) -> int: + row = conn.execute( + "delete from sessions where expires_at <= now() returning token_hash" + ).fetchall() + return len(row) + + +# --------------------------------------------------------------------------- # +# Durable cache invalidation queue (lab3) +# --------------------------------------------------------------------------- # +def enqueue_cache_invalidation(conn, token_hash: str) -> None: + """Record a session cache key that must be removed from Valkey. + + Written in the same transaction as the authoritative session delete, so a + committed logout can never lose the obligation to invalidate the cache. + """ + conn.execute( + """ + insert into cache_invalidations (token_hash, created_at) + values (%s, now()) + on conflict (token_hash) do nothing + """, + (token_hash,), + ) + + +def list_pending_invalidations(conn, limit: int) -> List[str]: + rows = conn.execute( + "select token_hash from cache_invalidations order by created_at limit %s", + (limit,), + ).fetchall() + return [str(row["token_hash"]) for row in rows] + + +def delete_cache_invalidations(conn, token_hashes: Sequence[str]) -> None: + if not token_hashes: + return + conn.execute( + "delete from cache_invalidations where token_hash = any(%s)", + (list(token_hashes),), + ) + + +# --------------------------------------------------------------------------- # +# Attachments (lab4): metadata in PostgreSQL, bytes in S3 +# --------------------------------------------------------------------------- # +def insert_attachment( + conn, + attachment_id: str, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + s3_key: str, + status: str, + error_code: Optional[str], + created_at: dt.datetime, + updated_at: dt.datetime, +) -> Dict: + return conn.execute( + """ + insert into attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + status, s3_key, error_code, created_at, updated_at) + values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + returning id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, status, s3_key, error_code, created_at, updated_at + """, + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + sha256, + status, + s3_key, + error_code, + created_at, + updated_at, + ), + ).fetchone() + + +def get_attachment(conn, attachment_id: str) -> Optional[Dict]: + return conn.execute( + """ + select id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, status, s3_key, error_code, created_at, updated_at + from attachments where id = %s + """, + (attachment_id,), + ).fetchone() + + +def get_attachment_for(conn, attachment_id: str, user_id: str) -> Optional[Dict]: + """Metadata only when the caller is a member of the attachment's chat.""" + return conn.execute( + """ + select a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.s3_key, a.error_code, + a.created_at, a.updated_at + from attachments a + where a.id = %s + and exists ( + select 1 from chat_members cm + where cm.chat_id = a.chat_id and cm.user_id = %s + ) + """, + (attachment_id, user_id), + ).fetchone() + + +def attachments_in_chat(conn, chat_id: str, attachment_ids: Sequence[str]) -> List[Dict]: + if not attachment_ids: + return [] + return conn.execute( + """ + select id, status from attachments + where chat_id = %s and id = any(%s::uuid[]) + """, + (chat_id, list(attachment_ids)), + ).fetchall() + + +def list_attachments_for_chat(conn, chat_id: str) -> List[Dict]: + return conn.execute( + "select id, s3_key, created_at from attachments where chat_id = %s", + (chat_id,), + ).fetchall() + + +def list_all_attachments(conn) -> List[Dict]: + return conn.execute("select id, s3_key, created_at from attachments").fetchall() + diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 00000000..fae9b4e7 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,57 @@ +"""Idempotent S3 bucket bootstrap (one-shot compose service). + +Creates the private attachments bucket with the root credentials before any API +replica starts. Re-running is safe; no credentials are printed. +""" +from __future__ import annotations + +import os +import time + +import boto3 +from botocore.client import Config +from botocore.exceptions import BotoCoreError, ClientError + + +def _read(env_key: str) -> str: + with open(os.environ[env_key], encoding="utf-8") as handle: + return handle.read().strip() + + +def main() -> None: + endpoint = os.environ["S3_ENDPOINT"] + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ["S3_BUCKET"] + client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=_read("S3_ROOT_ACCESS_KEY_ID_FILE"), + aws_secret_access_key=_read("S3_ROOT_SECRET_ACCESS_KEY_FILE"), + config=Config( + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 1}, + s3={"addressing_style": "path"}, + ), + ) + + last: Exception | None = None + for attempt in range(1, 31): + try: + try: + client.head_bucket(Bucket=bucket) + print(f"bucket {bucket} already present") + except ClientError: + client.create_bucket(Bucket=bucket) + print(f"bucket {bucket} created") + return + except (ClientError, BotoCoreError) as exc: + last = exc + print(f"bucket bootstrap not ready (attempt {attempt}/30)") + time.sleep(2) + raise SystemExit(f"bucket bootstrap failed: {last}") + + +if __name__ == "__main__": + main() diff --git a/src/s3_private_check.py b/src/s3_private_check.py new file mode 100644 index 00000000..a708a16f --- /dev/null +++ b/src/s3_private_check.py @@ -0,0 +1,37 @@ +"""Prove the bucket is private *outside* the API authorization path. + +Lists one object with the limited credentials, then requests the same object +over plain HTTP with NO credentials. A private bucket must answer AccessDenied +(403) or 404, never 200 with the bytes. + + docker compose exec -T api python -m src.s3_private_check +""" +from __future__ import annotations + +import urllib.error +import urllib.request + +from . import storage +from .config import settings + + +def main() -> None: + keys = [item["Key"] for item in storage.list_keys("")] + if not keys: + raise SystemExit("no objects to check; upload an attachment first") + key = keys[0] + url = f"{settings.s3_endpoint.rstrip('/')}/{settings.s3_bucket}/{key}" + try: + with urllib.request.urlopen(url, timeout=5) as response: + status = response.status + size = len(response.read()) + except urllib.error.HTTPError as error: + print(f"anonymous GET -> {error.code} (denied)") + assert error.code in {403, 404}, f"unexpected status {error.code}" + print("PASS: private bucket denies anonymous access") + return + raise SystemExit(f"FAIL: anonymous GET returned {status} with {size} bytes") + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..79aaa92d --- /dev/null +++ b/src/security.py @@ -0,0 +1,33 @@ +"""Password KDF (Argon2id) and opaque bearer-token helpers.""" +from __future__ import annotations + +import hashlib +import secrets + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError, VerificationError + +# Argon2id is the default of argon2-cffi's PasswordHasher. Parameters are kept +# modest so the local test suite stays fast while still using a real KDF. +_hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def new_token() -> str: + """Opaque bearer token with >= 256 bits of entropy.""" + return secrets.token_urlsafe(32) + + +def token_hash(token: str) -> str: + """Sessions are identified by a SHA-256 of the token, never the token itself.""" + return hashlib.sha256(token.encode("utf-8")).hexdigest() diff --git a/src/seed.py b/src/seed.py new file mode 100644 index 00000000..c59e338c --- /dev/null +++ b/src/seed.py @@ -0,0 +1,87 @@ +"""Synthetic data seeder for EXPLAIN evidence (never used by the public smoke). + +Runs INSIDE the api container so it can reach PostgreSQL over the compose +network: + + docker compose exec -T api python -m src.seed --messages 10000 + +It writes a small number of users/chats and one "hot" chat with many messages, +then prints the hot chat id so the operator can run EXPLAIN against it. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import uuid + +from .db import pool + + +def _now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Seed synthetic chat history") + parser.add_argument("--users", type=int, default=10) + parser.add_argument("--chats", type=int, default=3) + parser.add_argument("--messages", type=int, default=10000) + args = parser.parse_args() + + user_ids = [str(uuid.uuid4()) for _ in range(args.users)] + chat_ids = [str(uuid.uuid4()) for _ in range(args.chats)] + base = _now() + + with pool().connection() as conn: + with conn.cursor() as cur: + cur.executemany( + """ + insert into users (id, username, display_name, password_hash, created_at) + values (%s, %s, %s, %s, %s) + """, + [ + (uid, f"seed_{i}_{uuid.uuid4().hex[:6]}", f"Seed {i}", "seed-not-loginable", base) + for i, uid in enumerate(user_ids) + ], + ) + cur.executemany( + "insert into chats (id, title, created_at) values (%s, %s, %s)", + [(cid, f"Seed chat {i}", base) for i, cid in enumerate(chat_ids)], + ) + cur.executemany( + "insert into chat_members (chat_id, user_id) values (%s, %s)", + [(cid, uid) for cid in chat_ids for uid in user_ids], + ) + + hot_chat = chat_ids[0] + per_chat = max(1, args.messages // len(chat_ids)) + rows = [] + total = 0 + for chat_index, cid in enumerate(chat_ids): + count = args.messages - total if chat_index == len(chat_ids) - 1 else per_chat + count = max(0, count) + for _ in range(count): + rows.append( + ( + str(uuid.uuid4()), + cid, + user_ids[0], + "synthetic history message", + base, + ) + ) + total += count + cur.executemany( + """ + insert into messages (id, chat_id, sender_id, text, created_at) + values (%s, %s, %s, %s, %s) + """, + rows, + ) + + print(json.dumps({"hot_chat_id": hot_chat, "messages": args.messages, "chats": chat_ids})) + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..be6f0376 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,203 @@ +"""Sessions: PostgreSQL is the source of truth, Valkey is a shared session cache. + +Design (lab3): + * ``sessions`` rows in PostgreSQL decide validity (``expires_at > now()``) and + survive restarts with the named volume. + * Valkey caches ``session:{token_hash} -> {user_id, expires_at}`` for at most + ``SESSION_CACHE_TTL_SECONDS``. The TTL is a *bounded staleness* guarantee: + even if an invalidation is lost, a cached positive cannot outlive this + window. + * Logout deletes the PostgreSQL row and, in the same transaction, enqueues the + token hash in ``cache_invalidations``. A background drainer (see + ``src/invalidation.py``) removes the Valkey key and then the queue row, so + a logout that happened while Valkey was down is still honoured as soon as + the cache returns — instead of being resurrected from a stale entry. + * If Valkey is unreachable every cache operation degrades to PostgreSQL. The + API keeps serving; the cache is never trusted as the source of truth. + +Cache calls on the request path are bounded by a hard timeout and a small +circuit breaker (a stopped Valkey can make Docker DNS block for seconds). The +drainer bypasses the breaker on purpose: invalidation must not be skipped. +""" +from __future__ import annotations + +import concurrent.futures +import datetime as dt +import json +import logging +import time +from typing import Optional, Tuple + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +from . import repositories as repo +from . import security +from .config import settings + +logger = logging.getLogger("messenger.sessions") + +_client: Optional[redis.Redis] = None +_client_disabled = False + +_executor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="cache") +_OP_TIMEOUT = 0.5 +_BREAKER_SECONDS = 5.0 +_breaker_until = 0.0 + + +def _cache() -> Optional[redis.Redis]: + global _client, _client_disabled + if not settings.cache_enabled or _client_disabled: + return None + if _client is None: + try: + _client = redis.Redis.from_url( + settings.cache_url, + socket_timeout=0.2, + socket_connect_timeout=0.2, + health_check_interval=0, + retry_on_timeout=False, + retry=Retry(NoBackoff(), 0), + ) + except (ValueError, redis.RedisError): + logger.warning("valkey client init failed; cache disabled") + _client_disabled = True + return None + return _client + + +def _guard(fn, *args): + """Run a request-path cache operation with a deadline and a small breaker.""" + global _breaker_until + if time.monotonic() < _breaker_until: + return None + try: + return _executor.submit(fn, *args).result(timeout=_OP_TIMEOUT) + except (concurrent.futures.TimeoutError, redis.RedisError, OSError): + _breaker_until = time.monotonic() + _BREAKER_SECONDS + logger.warning("valkey unavailable; cache bypassed for %.0fs", _BREAKER_SECONDS) + return None + + +def _session_key(token_hash: str) -> str: + return f"session:{token_hash}" + + +def _cache_get(key: str) -> Optional[str]: + client = _cache() + if client is None: + return None + value = _guard(lambda: client.get(key)) + if value is None: + return None + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +def _cache_set(key: str, value: str, ttl_seconds: float) -> None: + client = _cache() + if client is None: + return + _guard(lambda: client.set(key, value, ex=max(1, int(ttl_seconds)))) + + +def _cache_delete(key: str) -> None: + client = _cache() + if client is None: + return + _guard(lambda: client.delete(key)) + + +def _encode_session(user_id: str, expires_at: dt.datetime) -> str: + return json.dumps( + {"user_id": user_id, "expires_at": repo.rfc3339(expires_at)}, + separators=(",", ":"), + ) + + +def _decode_session(raw: str) -> Optional[Tuple[str, dt.datetime]]: + try: + data = json.loads(raw) + user_id = str(data["user_id"]) + value = str(data["expires_at"]) + if value.endswith("Z"): + value = value[:-1] + "+00:00" + expires_at = dt.datetime.fromisoformat(value) + return user_id, expires_at + except (ValueError, KeyError, TypeError): + return None + + +def _cache_ttl(remaining: float) -> int: + return max(1, int(min(remaining, settings.session_cache_ttl_seconds))) + + +def create(conn, user_id: str) -> Tuple[str, dt.datetime]: + token = security.new_token() + hashed = security.token_hash(token) + expires_at = repo.utcnow() + dt.timedelta(seconds=settings.session_ttl_seconds) + repo.insert_session(conn, hashed, user_id, expires_at) + # The cache is populated by the first successful resolve() (read-through), + # so it can only ever contain sessions already committed to PostgreSQL. + return token, expires_at + + +def resolve(conn, token: str) -> Optional[str]: + """Return the session's user id, or None if the token is not valid. + + A cache hit is accepted only while the stored expiry is in the future and + only for the bounded cache TTL. A miss or any cache failure falls back to + the authoritative PostgreSQL row. + """ + hashed = security.token_hash(token) + cached = _cache_get(_session_key(hashed)) + if cached is not None: + decoded = _decode_session(cached) + if decoded is not None and decoded[1] > repo.utcnow(): + return decoded[0] + + row = repo.get_valid_session(conn, hashed) + if row is None: + # Remove any legacy/stale entry; revocation is already authoritative. + _cache_delete(_session_key(hashed)) + return None + + user_id = str(row["user_id"]) + remaining = (row["expires_at"] - repo.utcnow()).total_seconds() + if remaining > 0: + _cache_set( + _session_key(hashed), + _encode_session(user_id, row["expires_at"]), + _cache_ttl(remaining), + ) + return user_id + + +def revoke(conn, token: str) -> None: + hashed = security.token_hash(token) + repo.delete_session(conn, hashed) + if settings.cache_enabled: + # Durable obligation, committed together with the session delete. + repo.enqueue_cache_invalidation(conn, hashed) + # Best effort: the drainer retries if this fails (e.g. Valkey is down). + _cache_delete(_session_key(hashed)) + + +def invalidate_session_key(token_hash: str) -> bool: + """Delete one cache key for the drainer; True means it is safe to dequeue. + + Deliberately bypasses the circuit breaker: a transient cache outage must not + cause pending invalidations to be dropped. Socket timeouts still bound the + call so the worker cannot block forever. + """ + client = _cache() + if client is None: + # Cache deliberately disabled -> nothing to invalidate. A client that + # failed to initialise keeps its rows so a restart can retry. + return not settings.cache_enabled + try: + client.delete(_session_key(token_hash)) + return True + except (redis.RedisError, OSError): + return False diff --git a/src/state.py b/src/state.py new file mode 100644 index 00000000..276a5d95 --- /dev/null +++ b/src/state.py @@ -0,0 +1,19 @@ +"""Process-wide lifecycle flags. + +``shutting_down`` is raised from the SIGTERM/SIGINT handler before Uvicorn +stops accepting connections, so ``/health/ready`` reports ``503`` and the +balancer drains this replica instead of sending it new work. +""" +from __future__ import annotations + +import threading + +_shutting_down = threading.Event() + + +def begin_shutdown() -> None: + _shutting_down.set() + + +def is_shutting_down() -> bool: + return _shutting_down.is_set() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..ef223b83 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,119 @@ +"""S3-compatible object storage client (SeaweedFS). + +Only the API process uses these limited credentials; the bucket is private and +object keys are generated by the server, never derived from ``filename``. All +connectivity failures become a bounded ``503 unavailable``. + +Every operation runs under a hard deadline with a small circuit breaker: a +stopped S3 container can make Docker DNS itself block for seconds, and without +this guard a storage outage would stall request handling instead of failing fast. +""" +from __future__ import annotations + +import concurrent.futures +import logging +import time +from typing import BinaryIO, Iterable, Optional + +import boto3 +from botocore.client import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import settings +from .errors import ApiError + +logger = logging.getLogger("messenger.storage") + +_client = None + +_BOTO_CONFIG = Config( + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 1, "mode": "standard"}, + s3={"addressing_style": "path"}, +) + +_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="s3") +_BREAKER_SECONDS = 5.0 +_breaker_until = 0.0 + + +def _unavailable() -> ApiError: + return ApiError(503, "unavailable", "Хранилище недоступно") + + +def client(): + global _client + if _client is None: + _client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id or None, + aws_secret_access_key=settings.s3_secret_access_key or None, + config=_BOTO_CONFIG, + ) + return _client + + +def _run(fn, timeout: float): + """Run an S3 call under a deadline and a shared breaker.""" + global _breaker_until + if time.monotonic() < _breaker_until: + raise _unavailable() + try: + return _executor.submit(fn).result(timeout=timeout) + except (concurrent.futures.TimeoutError, BotoCoreError, ClientError, OSError): + _breaker_until = time.monotonic() + _BREAKER_SECONDS + logger.warning("S3 unavailable; operations fail fast for %.0fs", _BREAKER_SECONDS) + raise _unavailable() + + +def ready() -> bool: + """Return True if the bucket is reachable (used by /health/ready).""" + try: + _run(lambda: client().head_bucket(Bucket=settings.s3_bucket), timeout=1.5) + return True + except ApiError: + return False + + +def put_object(key: str, fileobj: BinaryIO, content_type: str) -> None: + _run( + lambda: client().upload_fileobj( + fileobj, settings.s3_bucket, key, ExtraArgs={"ContentType": content_type} + ), + timeout=30, + ) + + +def open_object(key: str): + """Return the boto3 get_object response (caller streams and closes Body).""" + + def _get(): + try: + return client().get_object(Bucket=settings.s3_bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: + raise ApiError(404, "not_found", "Объект не найден") + raise + + return _run(_get, timeout=10) + + +def delete_object(key: str) -> bool: + try: + _run(lambda: client().delete_object(Bucket=settings.s3_bucket, Key=key), timeout=10) + return True + except ApiError: + return False + + +def list_keys(prefix: str = "") -> Iterable[dict]: + paginator = client().get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix): + for item in page.get("Contents", []): + yield item + except (BotoCoreError, ClientError): + raise _unavailable() diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..17447d63 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,67 @@ +"""Student tests run against an already-running compose stand. + +Start the stand first (secrets + `docker compose up -d --build`), then: + + .venv\\Scripts\\python -m pytest student-tests -v + # or: make test-student PYTHON=.venv/bin/python + +Override the target with BASE_URL (default http://localhost:8080). +""" +from __future__ import annotations + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080") + + +@pytest.fixture(scope="session") +def base_url() -> str: + return BASE_URL.rstrip("/") + + +@pytest.fixture(scope="session") +def client(base_url): + from http_client import Client + + return Client(base_url) + + +def _register_and_login(client, label: str) -> dict: + suffix = os.urandom(4).hex() + username = f"s_{label}_{suffix}" + password = "T3st-" + os.urandom(8).hex() + registered = client.register(username, password, "Студент " + label) + assert registered.status == 201, registered.body + user = registered.json + session = client.login(username, password) + assert session.status == 201, session.body + payload = session.json + return { + "id": user["id"], + "username": username, + "password": password, + "user": user, + "token": payload["token"], + } + + +@pytest.fixture(scope="session") +def alice(client): + return _register_and_login(client, "alice") + + +@pytest.fixture(scope="session") +def bob(client): + return _register_and_login(client, "bob") + + +@pytest.fixture(scope="session") +def chat(client, alice, bob): + response = client.create_chat(alice["token"], [alice["id"], bob["id"]]) + assert response.status == 201, response.body + return response.json diff --git a/student-tests/http_client.py b/student-tests/http_client.py new file mode 100644 index 00000000..0622ffef --- /dev/null +++ b/student-tests/http_client.py @@ -0,0 +1,148 @@ +"""Minimal stdlib HTTP client for student black-box tests (no extra deps).""" +from __future__ import annotations + +import base64 +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Response: + status: int + headers: object + body: bytes + + @property + def json(self): + if not self.body: + return None + return json.loads(self.body.decode("utf-8")) + + @property + def content_type(self) -> str: + return self.headers.get_content_type() + + @property + def request_id(self) -> Optional[str]: + return self.headers.get("X-Request-Id") + + @property + def instance_id(self) -> Optional[str]: + return self.headers.get("X-Instance-Id") + + +class Client: + def __init__(self, base_url: str): + self.base = base_url.rstrip("/") + + def request( + self, + method: str, + path: str, + body=None, + raw: Optional[bytes] = None, + token: Optional[str] = None, + basic: Optional[str] = None, + headers: Optional[dict] = None, + ) -> Response: + data = raw + request_headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if raw is not None: + request_headers.setdefault("Content-Type", "application/json") + if token is not None: + request_headers["Authorization"] = "Bearer " + token + if basic is not None: + request_headers["Authorization"] = "Basic " + basic + if headers: + request_headers.update(headers) + request = urllib.request.Request( + self.base + path, data=data, headers=request_headers, method=method + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return Response(response.status, response.headers, response.read()) + except urllib.error.HTTPError as error: + return Response(error.code, error.headers, error.read()) + + # -- convenience -------------------------------------------------------- + def register(self, username: str, password: str, display_name: str = "Студент") -> Response: + return self.request( + "POST", + "/api/v1/users", + body={"username": username, "display_name": display_name, "password": password}, + ) + + def login(self, username: str, password: str) -> Response: + credential = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode() + return self.request("POST", "/api/v1/auth/sessions", basic=credential) + + def me(self, token: str) -> Response: + return self.request("GET", "/api/v1/users/me", token=token) + + def create_chat(self, token: str, member_ids, title: str = "Чат") -> Response: + return self.request( + "POST", + "/api/v1/chats", + body={"title": title, "member_ids": list(member_ids)}, + token=token, + ) + + def send( + self, + token: str, + chat_id: str, + text: str, + key: Optional[str] = None, + headers: Optional[dict] = None, + attachments=None, + ) -> Response: + # lab3 requires an Idempotency-Key on every message POST. + request_headers = dict(headers or {}) + request_headers.setdefault("Idempotency-Key", key or os.urandom(16).hex()) + body = {"text": text} + if attachments is not None: + body["attachment_ids"] = list(attachments) + return self.request( + "POST", + f"/api/v1/chats/{chat_id}/messages", + body=body, + token=token, + headers=request_headers, + ) + + def upload_attachment(self, token: str, chat_id: str, content: bytes, filename: str = "sample.png", + content_type: str = "image/png") -> Response: + return self.request( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={filename}", + raw=content, + token=token, + headers={"Content-Type": content_type}, + ) + + def get_attachment(self, token: str, attachment_id: str) -> Response: + return self.request("GET", f"/api/v1/attachments/{attachment_id}", token=token) + + def download(self, token: str, attachment_id: str, variant: Optional[str] = None) -> Response: + path = f"/api/v1/attachments/{attachment_id}/content" + if variant is not None: + path += f"?variant={variant}" + return self.request("GET", path, token=token) + + def list_messages(self, token: str, chat_id: str, limit: int = 50, cursor=None) -> Response: + path = f"/api/v1/chats/{chat_id}/messages?limit={limit}" + if cursor is not None: + from urllib.parse import quote + + path += "&cursor=" + quote(cursor, safe="") + return self.request("GET", path, token=token) + + def logout(self, token: str) -> Response: + return self.request("DELETE", "/api/v1/auth/sessions/current", token=token) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..88745108 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,132 @@ +"""Attachment tests for lab4 (grade 4): roundtrip, access control, limits.""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def _assert_error(response, status, code): + assert response.status == status, (response.status, response.body) + assert response.content_type == "application/json" + payload = response.json + assert payload["error"]["code"] == code + assert payload["error"]["request_id"] == response.request_id + + +def _fresh_actor(client, label): + from conftest import _register_and_login + + return _register_and_login(client, label) + + +def _peer_chat(client, actor): + peer = _fresh_actor(client, "att_peer") + response = client.create_chat(actor["token"], [actor["id"], peer["id"]], title="Attachment chat") + assert response.status == 201, response.body + return response.json, peer + + +def test_upload_download_roundtrip(client, alice, bob, chat): + upload = client.upload_attachment(alice["token"], chat["id"], SAMPLE, filename="sample.png") + assert upload.status == 201, upload.body + att = upload.json + assert att["status"] == "ready" + assert att["error_code"] is None + assert att["variants"] == [] + assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] + assert att["content_type"] == "image/png" + assert att["size_bytes"] == len(SAMPLE) + assert att["sha256"] == hashlib.sha256(SAMPLE).hexdigest() + + meta = client.get_attachment(bob["token"], att["id"]) + assert meta.status == 200, meta.body + assert meta.json["id"] == att["id"] + + download = client.download(bob["token"], att["id"]) + assert download.status == 200, download.body + assert download.body == SAMPLE + assert download.content_type == "image/png" + assert download.headers.get("Location") is None, "download must not redirect" + + # A message may reference the attachment of its own chat, even with empty text. + message = client.send(alice["token"], chat["id"], "", attachments=[att["id"]]) + assert message.status == 201, message.body + assert message.json["attachment_ids"] == [att["id"]] + assert message.json["text"] == "" + + +def test_foreign_actor_gets_404(client, alice, bob, chat): + att = client.upload_attachment(alice["token"], chat["id"], SAMPLE).json + stranger = _fresh_actor(client, "foreign") + _assert_error(client.get_attachment(stranger["token"], att["id"]), 404, "not_found") + _assert_error(client.download(stranger["token"], att["id"]), 404, "not_found") + + +def test_cross_chat_attachment_is_404(client, alice, bob, chat): + other_chat, peer = _peer_chat(client, alice) + foreign = client.upload_attachment(alice["token"], other_chat["id"], SAMPLE).json + # Referencing an attachment that belongs to another chat is not allowed. + response = client.send(alice["token"], chat["id"], "", attachments=[foreign["id"]]) + _assert_error(response, 404, "not_found") + + +def test_oversized_upload_is_413(client, alice, chat): + too_big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10 * 1024 * 1024 + 1) + response = client.upload_attachment(alice["token"], chat["id"], too_big) + _assert_error(response, 413, "payload_too_large") + + +def test_corrupt_image_is_415(client, alice, chat): + corrupt = b"\x89PNG\r\n\x1a\n" + os.urandom(256) + response = client.upload_attachment(alice["token"], chat["id"], corrupt) + _assert_error(response, 415, "unsupported_media_type") + + +def test_mime_spoof_is_415(client, alice, chat): + # GIF bytes claimed to be a PNG: the real format decides. + spoof = b"GIF89a" + os.urandom(64) + response = client.upload_attachment( + alice["token"], chat["id"], spoof, content_type="image/png" + ) + _assert_error(response, 415, "unsupported_media_type") + + +def test_empty_upload_is_415(client, alice, chat): + _assert_error(client.upload_attachment(alice["token"], chat["id"], b""), 415, "unsupported_media_type") + + +def test_unsafe_filename_is_400(client, alice, chat): + _assert_error( + client.upload_attachment(alice["token"], chat["id"], SAMPLE, filename="a/b.png"), + 400, + "invalid_request", + ) + + +def test_empty_text_without_attachment_is_400(client, alice, chat): + _assert_error(client.send(alice["token"], chat["id"], ""), 400, "invalid_request") + + +def test_message_idempotency_includes_attachments(client, alice, chat): + att = client.upload_attachment(alice["token"], chat["id"], SAMPLE).json + key = os.urandom(16).hex() + first = client.send(alice["token"], chat["id"], "with file", key=key, attachments=[att["id"]]) + assert first.status == 201, first.body + replay = client.send(alice["token"], chat["id"], "with file", key=key, attachments=[att["id"]]) + assert replay.status == 201, replay.body + assert replay.json == first.json, "idempotent replay differs" + # Same key, different body -> conflict. + from conftest import _register_and_login # noqa: F401 (kept for symmetry) + + conflict = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "different"}, + token=alice["token"], + headers={"Idempotency-Key": key}, + ) + _assert_error(conflict, 409, "conflict") diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..646db383 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,122 @@ +"""Authentication and registration tests (lab2 auth, unchanged in lab3).""" +from __future__ import annotations + +import base64 +import os + + +def _assert_error(response, status, code, challenge_prefix=None): + assert response.status == status, (response.status, response.body) + assert response.content_type == "application/json" + payload = response.json + assert payload["error"]["code"] == code + assert response.request_id, "missing X-Request-Id" + assert payload["error"]["request_id"] == response.request_id + if challenge_prefix: + header = response.headers.get("WWW-Authenticate", "") + assert header.lower().startswith(challenge_prefix.lower()), header + + +def test_registration_requires_password(client): + response = client.request( + "POST", "/api/v1/users", body={"username": "no_pw_" + os.urandom(3).hex(), "display_name": "X"} + ) + _assert_error(response, 400, "invalid_request") + + +def test_short_password_is_400(client): + response = client.register("shortpw_" + os.urandom(3).hex(), "too-short") + _assert_error(response, 400, "invalid_request") + + +def test_extra_field_is_400(client): + response = client.request( + "POST", + "/api/v1/users", + body={ + "username": "extra_" + os.urandom(3).hex(), + "display_name": "X", + "password": "T3st-" + os.urandom(8).hex(), + "role": "admin", + }, + ) + _assert_error(response, 400, "invalid_request") + + +def test_malformed_json_is_400(client): + response = client.request("POST", "/api/v1/users", raw=b"{not json") + _assert_error(response, 400, "invalid_request") + + +def test_duplicate_username_is_409(client): + username = "dup_" + os.urandom(4).hex() + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + _assert_error(client.register(username, password), 409, "conflict") + + +def test_registration_never_returns_password(client): + response = client.register("pw_hidden_" + os.urandom(3).hex(), "T3st-" + os.urandom(8).hex()) + assert response.status == 201, response.body + assert "password" not in response.json + + +def test_wrong_password_is_401_basic(client, alice): + _assert_error( + client.login(alice["username"], "definitely-wrong-password"), + 401, + "unauthorized", + challenge_prefix="Basic", + ) + + +def test_unknown_user_is_401_basic(client): + _assert_error( + client.login("ghost_" + os.urandom(4).hex(), "T3st-" + os.urandom(8).hex()), + 401, + "unauthorized", + challenge_prefix="Basic", + ) + + +def test_basic_on_business_endpoint_is_401_bearer(client, alice): + credential = base64.b64encode(f"{alice['username']}:{alice['password']}".encode()).decode() + response = client.request("GET", "/api/v1/users/me", basic=credential) + _assert_error(response, 401, "unauthorized", challenge_prefix="Bearer") + + +def test_missing_auth_is_401_bearer(client): + _assert_error(client.request("GET", "/api/v1/users/me"), 401, "unauthorized", challenge_prefix="Bearer") + + +def test_unknown_bearer_is_401(client): + _assert_error( + client.me("not-a-real-token-" + os.urandom(8).hex()), + 401, + "unauthorized", + challenge_prefix="Bearer", + ) + + +def test_x_user_id_is_ignored(client, alice, bob): + only_header = client.request("GET", "/api/v1/users/me", headers={"X-User-Id": alice["id"]}) + _assert_error(only_header, 401, "unauthorized", challenge_prefix="Bearer") + response = client.request( + "GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]} + ) + assert response.status == 200, response.body + assert response.json["id"] == alice["id"] + + +def test_logout_revokes_session(client): + suffix = os.urandom(4).hex() + username = f"s_out_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + assert client.me(token).status == 200 + logout = client.logout(token) + assert logout.status == 204, logout.body + assert not logout.body + _assert_error(client.me(token), 401, "unauthorized", challenge_prefix="Bearer") + _assert_error(client.logout(token), 401, "unauthorized", challenge_prefix="Bearer") diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..a6506194 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,37 @@ +"""Concurrency: 100 parallel unique messages survive without loss.""" +from __future__ import annotations + +import concurrent.futures +import os + + +def _register(client, label): + username = f"s_race_{label}_{os.urandom(4).hex()}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + return {"id": client.me(token).json["id"], "token": token} + + +def test_100_concurrent_messages_are_not_lost(client, alice): + senders = [_register(client, str(i)) for i in range(10)] + members = [alice["id"]] + [sender["id"] for sender in senders] + chat = client.create_chat(alice["token"], members).json + + def worker(task): + sender, index = task + response = client.send(sender["token"], chat["id"], f"race-{index}") + assert response.status == 201, response.body + return response.json["id"] + + tasks = [(senders[index // 10], index) for index in range(100)] + with concurrent.futures.ThreadPoolExecutor(max_workers=25) as pool: + created = list(pool.map(worker, tasks)) + + assert len(created) == 100 + assert len(set(created)) == 100, "duplicate message ids" + history = client.list_messages(alice["token"], chat["id"], limit=100).json + stored = [item["id"] for item in history["items"]] + assert len(stored) == 100, f"stored {len(stored)} of 100" + assert len(set(stored)) == 100 + assert set(stored) == set(created) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..9e667e5e --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,65 @@ +"""Lab3 shared idempotency: replay, conflict and cross-replica concurrency.""" +from __future__ import annotations + +import concurrent.futures +import json +import uuid + + +def test_replay_same_key_returns_same_message(client, alice, chat): + key = uuid.uuid4().hex + first = client.send(alice["token"], chat["id"], "идемпотентно", key=key) + assert first.status == 201, first.body + replay = client.send(alice["token"], chat["id"], "идемпотентно", key=key) + assert replay.status == 201, replay.body + assert replay.json == first.json, "replay must return the same Message" + + +def test_same_key_different_body_is_409(client, alice, chat): + key = uuid.uuid4().hex + assert client.send(alice["token"], chat["id"], "первый", key=key).status == 201 + conflict = client.send(alice["token"], chat["id"], "второй", key=key) + assert conflict.status == 409, conflict.body + assert conflict.json["error"]["code"] == "conflict" + + +def test_missing_key_is_400(client, alice, chat): + response = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "без ключа"}, + token=alice["token"], + ) + assert response.status == 400, response.body + assert response.json["error"]["code"] == "invalid_request" + + +def test_invalid_key_is_400(client, alice, chat): + for bad in ("bad key", "has/slash", "x" * 129, "with$sign"): + response = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "плохой ключ"}, + token=alice["token"], + headers={"Idempotency-Key": bad}, + ) + assert response.status == 400, (bad, response.status, response.body) + + +def test_concurrent_same_key_creates_exactly_one_message(client, alice, chat): + key = uuid.uuid4().hex + text = "конкурентный повтор " + uuid.uuid4().hex + + def send(_): + return client.send(alice["token"], chat["id"], text, key=key) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(send, range(8))) + + assert all(r.status == 201 for r in responses), [r.status for r in responses] + bodies = {json.dumps(r.json, sort_keys=True) for r in responses} + assert len(bodies) == 1, "parallel replays returned different messages" + + history = client.list_messages(alice["token"], chat["id"], limit=100).json["items"] + matching = [m for m in history if m["text"] == text] + assert len(matching) == 1, f"expected exactly one stored message, got {len(matching)}" diff --git a/student-tests/test_instances.py b/student-tests/test_instances.py new file mode 100644 index 00000000..d8cf51ef --- /dev/null +++ b/student-tests/test_instances.py @@ -0,0 +1,23 @@ +"""Lab3 balancing: X-Instance-Id is present and more than one replica serves.""" +from __future__ import annotations + + +def test_instance_header_present_on_success(client, alice): + response = client.me(alice["token"]) + assert response.status == 200, response.body + assert response.instance_id, "missing X-Instance-Id" + + +def test_instance_header_present_on_error(client): + response = client.request("GET", "/api/v1/users/me") + assert response.status == 401, response.body + assert response.instance_id, "error response missing X-Instance-Id" + + +def test_at_least_two_instances_seen_through_lb(client, alice): + seen = set() + for _ in range(60): + instance = client.me(alice["token"]).instance_id + if instance: + seen.add(instance) + assert len(seen) >= 2, f"expected >=2 API replicas behind the LB, saw {sorted(seen)}" diff --git a/student-tests/test_negative.py b/student-tests/test_negative.py new file mode 100644 index 00000000..a5af2237 --- /dev/null +++ b/student-tests/test_negative.py @@ -0,0 +1,65 @@ +"""Negative and access-control tests (carried over, lab3 keeps them).""" +from __future__ import annotations + +import os + + +def _assert_error(response, status, code): + assert response.status == status, (response.status, response.body) + payload = response.json + assert payload["error"]["code"] == code + assert payload["error"]["request_id"] == response.request_id + + +def _fresh_actor(client, label="peer"): + suffix = os.urandom(4).hex() + username = f"s_{label}_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + return {"id": client.me(token).json["id"], "token": token} + + +def test_foreign_chat_is_404(client, alice, chat): + stranger = _fresh_actor(client, "stranger") + assert client.request("GET", f"/api/v1/chats/{chat['id']}", token=stranger["token"]).status == 404 + _assert_error(client.send(stranger["token"], chat["id"], "чужое"), 404, "not_found") + _assert_error(client.list_messages(stranger["token"], chat["id"]), 404, "not_found") + + +def test_empty_message_is_400(client, alice, chat): + for text in ("", " ", "\t\n"): + _assert_error(client.send(alice["token"], chat["id"], text), 400, "invalid_request") + + +def test_oversized_message_is_400(client, alice, chat): + _assert_error(client.send(alice["token"], chat["id"], "x" * 4001), 400, "invalid_request") + + +def test_limit_out_of_range_is_400(client, alice, chat): + _assert_error(client.list_messages(alice["token"], chat["id"], limit=0), 400, "invalid_request") + _assert_error(client.list_messages(alice["token"], chat["id"], limit=101), 400, "invalid_request") + + +def test_sender_id_in_body_is_rejected(client, alice, bob, chat): + response = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "привет", "sender_id": bob["id"]}, + token=alice["token"], + headers={"Idempotency-Key": os.urandom(12).hex()}, + ) + _assert_error(response, 400, "invalid_request") + + +def test_chat_without_caller_is_400(client, bob, alice): + _assert_error(client.create_chat(bob["token"], [alice["id"]]), 400, "invalid_request") + + +def test_chat_with_unknown_member_is_400(client, alice): + unknown = "00000000-0000-4000-8000-000000000000" + _assert_error(client.create_chat(alice["token"], [alice["id"], unknown]), 400, "invalid_request") + + +def test_chat_with_duplicate_members_is_400(client, alice): + _assert_error(client.create_chat(alice["token"], [alice["id"], alice["id"]]), 400, "invalid_request") diff --git a/student-tests/test_pagination.py b/student-tests/test_pagination.py new file mode 100644 index 00000000..18e7188b --- /dev/null +++ b/student-tests/test_pagination.py @@ -0,0 +1,79 @@ +"""Keyset pagination tests (carried over from lab2).""" +from __future__ import annotations + +import base64 +import json +import os + + +def _fake_cursor(payload) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _fresh_chat(client, alice): + suffix = os.urandom(4).hex() + username = f"s_page_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + peer_token = client.login(username, password).json["token"] + peer_id = client.me(peer_token).json["id"] + response = client.create_chat(alice["token"], [alice["id"], peer_id]) + assert response.status == 201, response.body + return response.json + + +def test_full_traversal_without_duplicates(client, alice): + chat = _fresh_chat(client, alice) + expected = [] + for index in range(5): + response = client.send(alice["token"], chat["id"], f"msg-{index}") + assert response.status == 201, response.body + expected.append(response.json["id"]) + + seen = [] + cursor = None + while True: + page = client.list_messages(alice["token"], chat["id"], limit=2, cursor=cursor).json + assert len(page["items"]) <= 2 + seen.extend(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + assert len(seen) == 5 + assert len(set(seen)) == 5, "duplicate across pages" + assert seen == expected, "pagination reordered history" + + +def test_cursor_from_other_chat_is_400(client, alice): + first = _fresh_chat(client, alice) + second = _fresh_chat(client, alice) + for index in range(3): + client.send(alice["token"], first["id"], f"a{index}") + client.send(alice["token"], second["id"], f"b{index}") + cursor = client.list_messages(alice["token"], first["id"], limit=1).json["next_cursor"] + assert cursor + foreign = client.list_messages(alice["token"], second["id"], limit=1, cursor=cursor) + assert foreign.status == 400, foreign.body + assert foreign.json["error"]["code"] == "invalid_request" + + +def test_garbage_cursor_is_400(client, alice): + chat = _fresh_chat(client, alice) + client.send(alice["token"], chat["id"], "hello") + response = client.list_messages(alice["token"], chat["id"], cursor="!!!not-a-cursor!!!") + assert response.status == 400, response.body + + +def test_well_formed_cursor_with_wrong_types_is_400(client, alice): + bad_payloads = [ + {"r": "chats", "t": "2026-01-01T00:00:00.000000Z", "i": 1}, + {"r": "chats", "t": "2026-01-01T00:00:00.000000Z", "i": []}, + {"r": "chats", "t": "2026-01-01T00:00:00.000000Z", "i": True}, + {"r": "chats", "t": "not-a-time", "i": "11111111-1111-4111-8111-111111111111"}, + ] + for payload in bad_payloads: + response = client.request( + "GET", "/api/v1/chats?cursor=" + _fake_cursor(payload), token=alice["token"] + ) + assert response.status == 400, (payload, response.status, response.body) diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..a095feb0 --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,34 @@ +"""Grade-4 integrity checks: registration race and composite-op rollback.""" +from __future__ import annotations + +import concurrent.futures +import os + + +def test_concurrent_duplicate_username_single_winner(client): + username = "race_dup_" + os.urandom(4).hex() + password = "T3st-" + os.urandom(8).hex() + + def register(_): + return client.register(username, password).status + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool: + statuses = list(pool.map(register, range(5))) + + assert statuses.count(201) == 1, statuses + assert statuses.count(409) == 4, statuses + + +def test_failed_chat_creation_rolls_back(client, alice): + def chats_count(): + return len(client.request("GET", "/api/v1/chats?limit=100", token=alice["token"]).json["items"]) + + before = chats_count() + unknown = "00000000-0000-4000-8000-000000000000" + response = client.create_chat(alice["token"], [alice["id"], unknown], title="should-rollback") + assert response.status == 400, response.body + after = chats_count() + assert after == before, "failed composite operation left a partial chat" + + listed = client.request("GET", "/api/v1/chats?limit=100", token=alice["token"]).json["items"] + assert all(item["title"] != "should-rollback" for item in listed) diff --git a/student-tests/test_session_revocation.py b/student-tests/test_session_revocation.py new file mode 100644 index 00000000..e5c56547 --- /dev/null +++ b/student-tests/test_session_revocation.py @@ -0,0 +1,34 @@ +"""Lab3 revocation: logout takes effect immediately on every replica. + +Session validity is decided only by PostgreSQL, so a revoked token must be +rejected regardless of which replica answers and regardless of the cache state. +The heavier Valkey-outage reproduction lives in scripts/check-revocation.py. +""" +from __future__ import annotations + +import os + + +def _actor(client, label="revoke"): + suffix = os.urandom(4).hex() + username = f"s_{label}_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + return {"id": client.me(token).json["id"], "token": token} + + +def test_logout_invalidates_across_replicas(client): + actor = _actor(client) + seen = set() + for _ in range(20): + response = client.me(actor["token"]) + assert response.status == 200, response.body + seen.add(response.instance_id) + assert len(seen) >= 2, f"expected the LB to spread requests, saw {sorted(seen)}" + + logout = client.logout(actor["token"]) + assert logout.status == 204, logout.body + + for _ in range(20): + assert client.me(actor["token"]).status == 401