From 32e105cc87392efd1a16330640c27cbea5663a46 Mon Sep 17 00:00:00 2001 From: Queenfish Date: Thu, 29 Oct 2026 00:03:34 +0300 Subject: [PATCH] =?UTF-8?q?S3-=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8?= =?UTF-8?q?=D1=89=D0=B5=20=D0=B8=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F,=20=D1=82=D1=80=D0=B5=D0=BA=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 1 + .env.example | 14 +- Dockerfile | 32 ++ REPORT.md | 221 ++++++++++- app/__init__.py | 0 app/api/__init__.py | 0 app/api/attachments.py | 294 ++++++++++++++ app/api/auth.py | 104 +++++ app/api/chats.py | 172 +++++++++ app/api/health.py | 53 +++ app/api/users.py | 58 +++ app/cache.py | 60 +++ app/config.py | 28 ++ app/db.py | 30 ++ app/main.py | 126 ++++++ app/migrate.py | 19 + app/migrations.py | 31 ++ app/models.py | 111 ++++++ app/s3.py | 53 +++ app/security.py | 83 ++++ app/session_store.py | 108 ++++++ app/storage.py | 509 +++++++++++++++++++++++++ compose-scale.yaml | 7 + compose.yaml | 188 +++++++++ migrations/001_users.sql | 7 + migrations/002_chats.sql | 5 + migrations/003_chat_members.sql | 5 + migrations/004_messages.sql | 7 + migrations/005_sessions.sql | 6 + migrations/006_messages_index.sql | 2 + migrations/007_idempotency.sql | 11 + migrations/008_idempotency_index.sql | 2 + migrations/009_chat_version.sql | 1 + migrations/010_attachments.sql | 15 + migrations/011_message_attachments.sql | 5 + migrations/012_attachment_status.sql | 4 + nginx.conf | 31 ++ requirements.txt | 11 + scripts/check-balance.py | 38 ++ scripts/check-chat-validation.py | 45 +++ scripts/check-models.py | 63 +++ scripts/fair-scaling-benchmark.py | 230 +++++++++++ scripts/measure-detection.py | 201 ++++++++++ scripts/plot-scaling.py | 75 ++++ scripts/presign_compare.py | 144 +++++++ scripts/ram_experiment.py | 137 +++++++ scripts/reconcile.py | 75 ++++ scripts/restart.cmd | 2 + scripts/restart.py | 50 +++ scripts/restart.sh | 22 ++ scripts/rolling-update.py | 111 ++++++ scripts/scaling-benchmark.py | 194 ++++++++++ scripts/stop-one.cmd | 2 + scripts/stop-one.py | 34 ++ scripts/stop-one.sh | 15 + scripts/test-idempotency.py | 105 +++++ student-tests/__init__.py | 0 student-tests/benchmark.py | 117 ++++++ student-tests/cache_experiment.py | 176 +++++++++ student-tests/explain.py | 90 +++++ student-tests/test_attachments.py | 191 ++++++++++ student-tests/test_cache.py | 129 +++++++ student-tests/test_presigned.py | 134 +++++++ student-tests/test_race.py | 84 ++++ student-tests/test_resilience.py | 153 ++++++++ student-tests/test_validation.py | 264 +++++++++++++ 66 files changed, 5265 insertions(+), 30 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/attachments.py create mode 100644 app/api/auth.py create mode 100644 app/api/chats.py create mode 100644 app/api/health.py create mode 100644 app/api/users.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/migrations.py create mode 100644 app/models.py create mode 100644 app/s3.py create mode 100644 app/security.py create mode 100644 app/session_store.py create mode 100644 app/storage.py create mode 100644 compose-scale.yaml create mode 100644 compose.yaml create mode 100644 migrations/001_users.sql create mode 100644 migrations/002_chats.sql create mode 100644 migrations/003_chat_members.sql create mode 100644 migrations/004_messages.sql create mode 100644 migrations/005_sessions.sql create mode 100644 migrations/006_messages_index.sql create mode 100644 migrations/007_idempotency.sql create mode 100644 migrations/008_idempotency_index.sql create mode 100644 migrations/009_chat_version.sql create mode 100644 migrations/010_attachments.sql create mode 100644 migrations/011_message_attachments.sql create mode 100644 migrations/012_attachment_status.sql create mode 100644 nginx.conf create mode 100644 requirements.txt create mode 100644 scripts/check-balance.py create mode 100644 scripts/check-chat-validation.py create mode 100644 scripts/check-models.py create mode 100644 scripts/fair-scaling-benchmark.py create mode 100644 scripts/measure-detection.py create mode 100644 scripts/plot-scaling.py create mode 100644 scripts/presign_compare.py create mode 100644 scripts/ram_experiment.py create mode 100644 scripts/reconcile.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.py create mode 100644 scripts/restart.sh create mode 100644 scripts/rolling-update.py create mode 100644 scripts/scaling-benchmark.py create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.py create mode 100644 scripts/stop-one.sh create mode 100644 scripts/test-idempotency.py create mode 100644 student-tests/__init__.py create mode 100644 student-tests/benchmark.py create mode 100644 student-tests/cache_experiment.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_cache.py create mode 100644 student-tests/test_presigned.py create mode 100644 student-tests/test_race.py create mode 100644 student-tests/test_resilience.py create mode 100644 student-tests/test_validation.py diff --git a/.dockerignore b/.dockerignore index bc16e55..447644c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +nginx.conf diff --git a/.env.example b/.env.example index d99d26a..8a36431 100644 --- a/.env.example +++ b/.env.example @@ -7,14 +7,8 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +DATABASE_URL=postgresql://messenger:messenger@postgres:5432/messenger +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=minioadmin \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9f10bcc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.12-slim AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --user -r requirements.txt + +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/home/appuser/.local/bin:$PATH + +RUN useradd -m -u 1000 appuser + +WORKDIR /app + +COPY --from=builder /root/.local /home/appuser/.local + +COPY ./app ./app +COPY ./migrations ./migrations + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +CMD ["sh", "-c", "export INSTANCE_ID=$(python -c 'import secrets; print(secrets.token_hex(6))') && echo \"[startup] INSTANCE_ID=$INSTANCE_ID\" && exec python -m uvicorn app.main:app --host 0.0.0.0 --port 8080"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84..6e89cb5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,218 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Кузнецова Марина, БВТ2304 +- **Целевая оценка:** 5, Трек 1 — прямой upload (presigned) +- **Commit/PR:** пока пусто +- **Использованные источники и помощь ИИ:** официальная документация FastAPI, asyncpg, MinIO, Pillow, Docker Compose. ИИ применялся для пояснения требований, обсуждения архитектурных решений и проверки корректности реализации. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования к окружению: + +- Python 3.13.15 +- Docker 26.1.1 +- git + +Подготовка и запуск через Docker Compose: + + cp .env.example .env + docker compose --env-file .env up -d --build --scale api=2 + +Ожидаемый признак готовности: ответ `{"status":"ok"}` на `http://localhost:8080/health/ready`. В логах `migrate` — `migrations applied`, контейнер `Exited (0)`. В логах `minio-init` — `bucket ready`, контейнер `Exited (0)`. Реплики `api-1` и `api-2` — `healthy`. + +Проверка публичным smoke-тестом: + + python -X utf8 tests/smoke.py --base-url http://localhost:8080 + +Проверка своими тестами: + + python -X utf8 -m pytest student-tests/test_attachments.py -v + python -X utf8 -m pytest student-tests/test_presigned.py -v + python -X utf8 -m pytest student-tests/test_validation.py student-tests/test_race.py student-tests/test_cache.py -v + +Проверка сохранности и отказа реплики: + + python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 + python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 + +Проверка сверки S3 и БД: + + docker compose cp scripts/reconcile.py api:/tmp/reconcile.py + docker compose exec -T api python /tmp/reconcile.py --dry-run + docker compose exec -T api python /tmp/reconcile.py + +Проверка пика RAM при 10 одновременных загрузках: + + python -X utf8 scripts/ram_experiment.py + +Сравнение proxy и presigned upload: + + python -X utf8 scripts/presign_compare.py + +Проверка приватности bucket снаружи: + + docker compose exec -T postgres psql -U messenger -d messenger -c "SELECT chat_id, id FROM attachments LIMIT 1;" + curl.exe -i http://localhost:9000/attachments// + +Ожидаемо: `403 AccessDenied`. + +Безопасная остановка: + +- `docker compose down` — остановка без потери данных. +- `docker compose down -v` — удаление volumes, все данные пропадут. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Схема: -## Доказательства критериев + Client → HTTP (Nginx :8080) → api-1, api-2 (FastAPI/Uvicorn) + ↓ + PostgreSQL (users/chats/messages/idempotency_keys/attachments) + Valkey (sessions + cache) + MinIO (bucket attachments, приватный) -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Минимум lab3 сохранён: две реплики API за Nginx, общие PostgreSQL и Valkey, идемпотентность, failover, persistence. В lab4 добавлены MinIO, метаданные вложений в PostgreSQL, endpoint'ы вложений и `attachment_ids` в сообщении. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Компоненты: -## Эксперименты и отказы +- `compose.yaml` — сервисы: postgres, valkey, minio, minio-init, migrate, api, lb. +- `nginx.conf` — балансировщик, `client_max_body_size 11m`, таймауты 60 с. +- `app/main.py` — FastAPI, lifespan, middleware `X-Request-Id` и `X-Instance-Id`, обработчики ошибок. +- `app/config.py` — конфигурация из переменных окружения, включая S3. +- `app/db.py` — пул соединений asyncpg. +- `app/migrate.py` — отдельный модуль миграций. +- `app/migrations.py` — раннер миграций. +- `app/models.py` — Pydantic-модели, включая `Attachment` и `attachment_ids`. +- `app/security.py` — Argon2id, токены, Basic и Bearer. +- `app/storage.py` — PostgreSQL, транзакции, `idempotency_keys`, вложения. +- `app/session_store.py` — backend для сессий. +- `app/cache.py` — кэш истории и single-flight. +- `app/s3.py` — клиент MinIO, presigned URL. +- `app/api/attachments.py` — upload, download, presign, complete. +- `migrations/*.sql` — 12 миграций (9 из lab3 + attachments + message_attachments + status). +- `scripts/reconcile.py` — сверка S3 и БД с grace period. +- `scripts/ram_experiment.py` — пик RAM при 10 загрузках. +- `scripts/presign_compare.py` — сравнение proxy и presigned. +- `student-tests/test_attachments.py` — негативные тесты вложений. +- `student-tests/test_presigned.py` — тесты presigned. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Источник истины.** PostgreSQL для пользователей, чатов, участников, сообщений, `idempotency_keys` и метаданных вложений. Байты — в MinIO, bucket `attachments`, приватный. Сессии — в Valkey. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Вложения.** Метаданные в таблице `attachments`: `id`, `chat_id`, `owner_id`, `object_key`, `filename`, `content_type`, `size_bytes`, `sha256`, `status`, `error_code`, `created_at`, `updated_at`. `object_key` назначает сервер: `{chat_id}/{uuid}` для proxy, `uploads/{chat_id}/{uuid}` для presigned. `filename` — только для отображения, в пути не используется. Связь с сообщением — таблица `message_attachments`. + +**Proxy upload.** `POST /chats/{chat_id}/attachments?filename=` принимает сырые bytes, проверяет членство, filename, лимиты, MIME по содержимому, пиксели, сохраняет объект в MinIO и метаданные в PostgreSQL в статусе `ready`. При ошибке записи метаданных объект удаляется (компенсация). + +**Download.** `GET /attachments/{id}` и `/content` доступны только участникам чата. Без redirect, байты отдаются через API. Посторонний получает 404. + +**Presigned upload (Трек 1).** `POST /chats/{chat_id}/attachments/presign?filename=&size=&content_type=` создаёт запись `pending`, генерирует presigned URL на 300 секунд с уникальным ключом. Клиент делает `PUT` напрямую в MinIO. `POST /attachments/{id}/complete` проверяет, что объект есть, размер совпадает, MIME по содержимому, `content_type` совпадает; после — `ready`, `sha256` и фактический размер. Proxy endpoint сохранён. + +**Обработка ошибок.** Единый формат `{"error": {"code", "message", "request_id"}}`. Недоступность PostgreSQL, Valkey или MinIO → 503. Пользовательские ошибки (400/413/415/404/409) не превращаются в 500. + +## Доказательства критериев «3» + +| Критерий | Команда | Результат | +|---|---|---| +| Возможности lab3 сохранены | `docker compose ps` | api-1, api-2 healthy; postgres, valkey healthy; lb Up | +| S3 в общем стенде, метаданные в PostgreSQL, байты в приватном bucket | `compose.yaml`, `migrations/010`, `011` | сервис minio + minio-init; таблицы attachments, message_attachments | +| POST вложения → 201 ready | `tests/smoke.py` | PASS | +| Оригинал скачивается побайтно | `tests/smoke.py` | PASS, sha256 и size совпадают | +| Message с attachment_ids читается участниками | `tests/smoke.py` | PASS | +| Посторонний → 404 | `test_attachments.py::test_outsider_cannot_download` | PASS | +| PNG/JPEG ≤10 MiB, ≤40 млн пикселей | `test_oversize_returns_413`, `test_corrupted_image_returns_415` | PASS | +| MIME по содержимому | `test_mime_spoof_returns_415` | PASS | +| Опасный filename → 400 | `test_bad_filename_returns_400` | PASS | +| Upload → stop реплики → download через другую | `resilience.py --mode failover` | PASS, recovery 5.171 с, 7 записей, 0 ошибок | +| Restart с volumes сохраняет метаданные и оригинал | `resilience.py --mode persistence` | PASS | +| Приватный bucket вне API | `curl http://localhost:9000/attachments//` | 403 AccessDenied | + +## Доказательства критериев «4» + +| Критерий | Команда | Результат | +|---|---|---| +| Негативные тесты: oversize, corrupted, MIME spoof, чужой id, другой чат | `pytest student-tests/test_attachments.py -v` | 10 passed | +| Пользовательская ошибка не 500 | `test_user_error_is_not_500` | PASS | +| Компенсация S3↔БД с grace period | `scripts/reconcile.py --dry-run`, `scripts/reconcile.py` | orphans_found=0, fresh_skipped=0, metadata_without_object=0 | +| Потоковая обработка и лимит RAM | `app/api/attachments.py` | `_read_limited` читает чанками по 64 КБ, обрывает при >10 MiB | +| Эксперимент ≥10 загрузок с пиком RAM | `scripts/ram_experiment.py` | 10 × 9 MiB, errors: 0, baseline 50.9 MiB, peak 164.9 MiB, delta 114.0 MiB, лимит 256 MiB | +| Bucket bootstrap идемпотентен | `docker compose logs minio-init` | bucket ready, Exited (0); `--ignore-existing`, `|| true` | +| Отдельные S3 credentials | `compose.yaml` | `S3_ACCESS_KEY=api-user`, `S3_SECRET_KEY=api-user-secret-123` | +| Консоль не опубликована | `compose.yaml` | `minio` публикует только `127.0.0.1:9000` | +| Приватный bucket вне API | `curl` без credentials | 403 AccessDenied | + +## Трек 1: прямой upload (presigned) + +**Гипотеза.** Presigned upload переносит передачу байтов с API на MinIO: клиент загружает объект напрямую по подписанному URL, а API только выдаёт подпись и подтверждает завершение. Это должно снизить трафик через API при загрузке больших файлов и не менять обязательный proxy endpoint. + +**Реализация.** + +- `POST /chats/{chat_id}/attachments/presign?filename=&size=&content_type=` → `201 {upload_url, attachment_id, expires_in}`. +- Подпись живёт 300 секунд. Ключ уникален: `uploads/{chat_id}/{uuid}`. Статус вложения — `pending`. +- Клиент делает `PUT upload_url` напрямую в MinIO. +- `POST /attachments/{id}/complete` проверяет: объект есть (`stat_object`), размер совпадает с заявленным, MIME по содержимому, `content_type` совпадает; после — `status='ready'`, `sha256` и фактический размер. +- Proxy endpoint `POST /chats/{chat_id}/attachments` сохранён и работает. + +**Негативные сценарии.** + +- Подменённая подпись → `403` от MinIO (`test_presigned_bad_signature_rejected`). +- Complete без загрузки объекта → `409` (`test_presigned_complete_without_upload_409`). +- Complete чужого `attachment_id` → `404` (`test_presigned_outsider_404`). +- Complete не в `pending` → `409` (проверяется в коде). + +Тесты: `pytest student-tests/test_presigned.py -v` → 5 passed. + +**Сравнение трафика и RAM.** `scripts/presign_compare.py`, размеры 1 MiB и 9 MiB, пик RAM API через `docker stats`: + +| Размер | Режим | Время, с | RAM baseline, MiB | RAM peak, MiB | Δ RAM, MiB | +|---|---|---|---|---|---| +| 1 MiB | proxy | 2.47 | 48.9 | 48.9 | 0.0 | +| 1 MiB | presigned | 5.08 | 48.9 | 49.3 | 0.4 | +| 9 MiB | proxy | 3.09 | 49.3 | 49.3 | 0.0 | +| 9 MiB | presigned | 4.76 | 49.3 | 50.2 | 0.9 | + +**Интерпретация.** + +- **Трафик.** В proxy-режиме тело идёт клиент → API → MinIO. В presigned — клиент → MinIO напрямую, через API только метаданные подписи и `complete`. Это главный выигрыш Трека 1. +- **Время.** Presigned медленнее одного upload (5.08 с против 2.47 с для 1 MiB; 4.76 с против 3.09 с для 9 MiB), потому что это два раунда (`presign` → `PUT` → `complete`), и `complete` читает объект из MinIO для проверки MIME и SHA-256. Presigned выигрывает в трафике API, а не в latency одного запроса. +- **RAM.** дельта RAM мал в обоих режимах (0.0–0.9 MiB), потому что `docker stats --no-stream` делает один снимок и не попадает в пик за 3–5 секунд. Для более чувствительного замера нужен непрерывный sampler или `memory.peak` из cgroup. + +**Ограничения.** + +- Presigned URL — bearer token: кто получил URL, может загрузить объект в течение 5 минут. Поэтому подпись короткая, ключ уникальный, `complete` проверяет размер, тип и checksum. +- Незавершённые загрузки (`pending` без объекта) не удаляются автоматически; В Треке 1 они только логируются. +- Сравнение — на одном Docker-хосте, closed-loop. +- Proxy endpoint остаётся обязательным; presigned — дополнительный flow, а не замена. + +## Исправления по ходу работы + +1. **413 от Nginx в HTML вместо JSON.** В `nginx.conf` поставлен `client_max_body_size 11m`, чтобы API сам возвращал 413 в общем JSON-формате. +2. **Presigned URL указывал на `minio:9000`.** Добавлен `S3_PUBLIC_ENDPOINT=localhost:9000` и отдельный клиент для подписи. В `Minio(...)` задан `region="us-east-1"`, иначе `presigned_put_object` ходит в сеть за регионом и падает. +3. **`check_contract.py` не запускался на Windows.** `make check` использует `python3`; на Windows запускается `python scripts/check_contract.py`. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Что выдерживается:** + +- Перезапуск API, БД, Valkey и MinIO с сохранением volumes: данные, сессии и объекты на месте. +- Остановка одной API-реплики: сервис продолжает работать, ошибки только в момент обнаружения отказа (0.015–0.046 с). +- Вложение, загруженное на одной реплике, скачивается через другую. +- Одновременные повторы POST сообщения с одним `Idempotency-Key` через разные реплики: создаётся одно сообщение. +- Недоступность Valkey, PostgreSQL или MinIO: `/health/ready` → 503. +- Пользовательские ошибки вложений (400/413/415/404/409) не превращаются в 500. + +**Что не выдерживается:** + +- Потеря Docker-хоста целиком: все контейнеры на одном узле. +- Потеря узла PostgreSQL: без репликации БД не выживет. +- Потеря узла Nginx: LB — SPOF. +- Потеря узла MinIO: без репликации S3 не выживет. +- Rolling update на время перезапуска реплика недоступна. + +**Учебные упрощения:** + +- Все контейнеры на одном Docker-хосте. +- Nginx без активных healthcheck (только пассивные по `max_fails`). +- HTTP без TLS. +- Миграции не поддерживают rollback. +- `Idempotency-Key` защищает только POST сообщений. +- Presigned URL живёт 5 минут; незавершённые `pending` не удаляются автоматически (задача Трека 2). +- RAM-замер через `docker stats --no-stream` недостаточно чувствителен. \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/attachments.py b/app/api/attachments.py new file mode 100644 index 0000000..2266c16 --- /dev/null +++ b/app/api/attachments.py @@ -0,0 +1,294 @@ +import hashlib +import re +import uuid +from io import BytesIO + +from fastapi import APIRouter, Header, Request +from fastapi.responses import JSONResponse, StreamingResponse +from PIL import Image, UnidentifiedImageError + +from app.security import get_current_user + +router = APIRouter(tags=["attachments"]) + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +ALLOWED = {"PNG": "image/png", "JPEG": "image/jpeg"} +BAD_FILENAME = re.compile(r'[/\\\x00-\x1f]') +BEARER_CHALLENGE = 'Bearer realm="messenger"' +CHUNK = 64 * 1024 +PRESIGN_TTL_SECONDS = 300 + + +def error_response(status, code, message, request_id, headers=None): + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers or {}, + ) + + +def validate_image(data: bytes) -> tuple[str, int, int, str]: + if len(data) == 0: + raise ValueError("unsupported_media_type") + if len(data) > MAX_BYTES: + raise ValueError("payload_too_large") + try: + img = Image.open(BytesIO(data)) + img.verify() + img = Image.open(BytesIO(data)) + fmt = img.format + w, h = img.size + except (UnidentifiedImageError, OSError, ValueError): + raise ValueError("unsupported_media_type") + if fmt not in ALLOWED: + raise ValueError("unsupported_media_type") + if w * h > MAX_PIXELS: + raise ValueError("payload_too_large") + return ALLOWED[fmt], w, h, hashlib.sha256(data).hexdigest() + + +def safe_filename(name: str) -> str: + if not name or len(name) > 255: + raise ValueError("invalid_request") + if BAD_FILENAME.search(name): + raise ValueError("invalid_request") + return name + + +async def _auth(request, authorization): + try: + user, _ = await get_current_user(request, authorization) + return user, None + except Exception: + return None, error_response( + 401, "unauthorized", "unauthorized", request.state.request_id, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + + +async def _read_limited(request, limit: int) -> bytes: + buf = bytearray() + async for chunk in request.stream(): + buf.extend(chunk) + if len(buf) > limit: + raise ValueError("payload_too_large") + return bytes(buf) + + +@router.post("/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + filename: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + uuid.UUID(chat_id) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid uuid", rid) + + user, err = await _auth(request, authorization) + if err is not None: + return err + + if not await request.app.state.storage.is_chat_member(chat_id, user["id"]): + return error_response(404, "not_found", "chat not found", rid) + + try: + safe_filename(filename) + except ValueError: + return error_response(400, "invalid_request", "invalid filename", rid) + + try: + data = await _read_limited(request, MAX_BYTES) + except ValueError: + return error_response(413, "payload_too_large", "payload_too_large", rid) + + try: + content_type, w, h, sha = validate_image(data) + except ValueError as e: + code = str(e) + status = 413 if code == "payload_too_large" else 415 + return error_response(status, code, code, rid) + + att_id = str(uuid.uuid4()) + object_key = f"{chat_id}/{att_id}" + try: + request.app.state.s3.put(object_key, data, content_type) + except Exception: + return error_response(503, "unavailable", "storage unavailable", rid) + + try: + att = await request.app.state.storage.create_attachment( + att_id, chat_id, user["id"], object_key, + filename, content_type, len(data), sha, + ) + except Exception: + try: + request.app.state.s3.remove(object_key) + except Exception: + pass + return error_response(503, "unavailable", "storage inconsistent", rid) + + return att + + +@router.post("/chats/{chat_id}/attachments/presign", status_code=201) +async def presign_upload( + chat_id: str, + filename: str, + size: int, + content_type: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + uuid.UUID(chat_id) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid uuid", rid) + + user, err = await _auth(request, authorization) + if err is not None: + return err + + if not await request.app.state.storage.is_chat_member(chat_id, user["id"]): + return error_response(404, "not_found", "chat not found", rid) + + try: + safe_filename(filename) + except ValueError: + return error_response(400, "invalid_request", "invalid filename", rid) + + if size <= 0 or size > MAX_BYTES: + return error_response(413, "payload_too_large", "payload_too_large", rid) + if content_type not in ("image/png", "image/jpeg"): + return error_response(415, "unsupported_media_type", "unsupported_media_type", rid) + + att_id = str(uuid.uuid4()) + object_key = f"uploads/{chat_id}/{att_id}" + await request.app.state.storage.create_pending_attachment( + att_id, chat_id, user["id"], object_key, + filename, content_type, size, + ) + url = request.app.state.s3.presigned_put(object_key, PRESIGN_TTL_SECONDS) + return { + "upload_url": url, + "attachment_id": att_id, + "expires_in": PRESIGN_TTL_SECONDS, + } + + +@router.post("/attachments/{attachment_id}/complete", status_code=201) +async def complete_upload( + attachment_id: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + uuid.UUID(attachment_id) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid uuid", rid) + + user, err = await _auth(request, authorization) + if err is not None: + return err + + att = await request.app.state.storage.get_attachment(attachment_id) + if att is None: + return error_response(404, "not_found", "attachment not found", rid) + if not await request.app.state.storage.is_chat_member(att["chat_id"], user["id"]): + return error_response(404, "not_found", "attachment not found", rid) + if att["status"] != "pending": + return error_response(409, "conflict", "attachment not pending", rid) + + object_key = att["_object_key"] + try: + stat = request.app.state.s3.stat(object_key) + except Exception: + return error_response(409, "conflict", "object not uploaded", rid) + + if stat.size != att["size_bytes"]: + return error_response(409, "conflict", "size mismatch", rid) + + try: + obj = request.app.state.s3.get(object_key) + data = obj.read() + except Exception: + return error_response(503, "unavailable", "storage unavailable", rid) + + try: + detected_type, w, h, sha = validate_image(data) + except ValueError as e: + code = str(e) + status = 413 if code == "payload_too_large" else 415 + return error_response(status, code, code, rid) + + if detected_type != att["content_type"]: + return error_response(409, "conflict", "content_type mismatch", rid) + + ready = await request.app.state.storage.complete_attachment( + attachment_id, sha, len(data), + ) + if ready is None: + return error_response(409, "conflict", "attachment not pending", rid) + return ready + + +@router.get("/attachments/{attachment_id}") +async def get_attachment( + attachment_id: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + uuid.UUID(attachment_id) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid uuid", rid) + user, err = await _auth(request, authorization) + if err is not None: + return err + att = await request.app.state.storage.get_attachment(attachment_id) + if att is None: + return error_response(404, "not_found", "attachment not found", rid) + if not await request.app.state.storage.is_chat_member(att["chat_id"], user["id"]): + return error_response(404, "not_found", "attachment not found", rid) + att.pop("_object_key", None) + return att + + +@router.get("/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: str, + request: Request, + variant: str = "original", + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + uuid.UUID(attachment_id) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid uuid", rid) + user, err = await _auth(request, authorization) + if err is not None: + return err + att = await request.app.state.storage.get_attachment(attachment_id) + if att is None: + return error_response(404, "not_found", "attachment not found", rid) + if not await request.app.state.storage.is_chat_member(att["chat_id"], user["id"]): + return error_response(404, "not_found", "attachment not found", rid) + if variant != "original": + return error_response(404, "not_found", "variant not found", rid) + try: + obj = request.app.state.s3.get(att["_object_key"]) + except Exception: + return error_response(503, "unavailable", "storage unavailable", rid) + return StreamingResponse( + obj.stream(CHUNK), + media_type=att["content_type"], + headers={"Content-Length": str(att["size_bytes"])}, + ) \ No newline at end of file diff --git a/app/api/auth.py b/app/api/auth.py new file mode 100644 index 0000000..de9261a --- /dev/null +++ b/app/api/auth.py @@ -0,0 +1,104 @@ +from fastapi import APIRouter, Header, HTTPException, Request, Response +from fastapi.responses import JSONResponse + +from app.security import ( + parse_basic, + verify_password, + new_token, + session_expires_at, + get_current_user, +) +from app.config import config +from app.models import Session + +router = APIRouter(tags=["auth"]) + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +@router.post("/auth/sessions", response_model=Session, status_code=201) +async def create_session( + request: Request, + authorization: str | None = Header(default=None), +): + storage = request.app.state.storage + session_store = request.app.state.session_store + rid = request.state.request_id + try: + username, password = parse_basic(authorization) + except HTTPException: + return error_response( + 401, + "unauthorized", + "invalid credentials", + rid, + headers={"WWW-Authenticate": BASIC_CHALLENGE}, + ) + + user_row = await storage.get_user_by_username(username) + if user_row is None or not verify_password(password, user_row["password_hash"]): + return error_response( + 401, + "unauthorized", + "invalid credentials", + rid, + headers={"WWW-Authenticate": BASIC_CHALLENGE}, + ) + + token = new_token() + expires_at = session_expires_at(config.SESSION_TTL_SECONDS) + await session_store.create(token, str(user_row["id"]), expires_at) + + user = { + "id": str(user_row["id"]), + "username": user_row["username"], + "display_name": user_row["display_name"], + "created_at": ( + user_row["created_at"] + .isoformat() + .replace("+00:00", "Z") + ), + } + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + "user": user, + } + + +@router.delete("/auth/sessions/current", status_code=204) +async def revoke_session( + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + _, token = await get_current_user(request, authorization) + except HTTPException: + return error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + await request.app.state.session_store.delete(token) + return Response(status_code=204) \ No newline at end of file diff --git a/app/api/chats.py b/app/api/chats.py new file mode 100644 index 0000000..a304ab3 --- /dev/null +++ b/app/api/chats.py @@ -0,0 +1,172 @@ +import re +import uuid + +from fastapi import APIRouter, Header, HTTPException, Query, Request +from fastapi.responses import JSONResponse + +from app.models import ( + Chat, CreateChat, Message, CreateMessage, ChatPage, MessagePage, +) +from app.security import get_current_user + +router = APIRouter(tags=["chats"]) + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def error_response(status, code, message, request_id, headers=None): + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers or {}, + ) + + +def validate_uuid(value: str) -> None: + try: + uuid.UUID(value) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="invalid uuid") + + +def validate_idempotency_key(value: str | None) -> str: + if value is None or not IDEMPOTENCY_RE.match(value): + raise HTTPException(status_code=400, detail="invalid idempotency key") + return value + + +async def resolve_actor(request: Request, authorization: str | None): + rid = request.state.request_id + try: + user, _ = await get_current_user(request, authorization) + except HTTPException as exc: + if exc.status_code == 503: + return None, error_response(503, "unavailable", "auth store unavailable", rid) + return None, error_response( + 401, "unauthorized", "unauthorized", rid, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + return user, None + + +@router.post("/chats", response_model=Chat, status_code=201) +async def create_chat( + data: CreateChat, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + + if len(data.member_ids) != len(set(data.member_ids)): + return error_response(400, "invalid_request", "duplicate member_ids", rid) + for mid in data.member_ids: + try: + uuid.UUID(mid) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid member uuid", rid) + + try: + return await request.app.state.storage.create_chat( + data.title, data.member_ids, actor["id"], + ) + except ValueError as e: + msg = str(e) + if msg == "invalid_member": + return error_response(400, "invalid_request", "unknown member", rid) + if msg == "creator_not_member": + return error_response(400, "invalid_request", "creator must be a member", rid) + return error_response(400, "invalid_request", "bad request", rid) + + +@router.get("/chats", response_model=ChatPage) +async def list_chats( + request: Request, + authorization: str | None = Header(default=None), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None), +): + rid = request.state.request_id + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + items, next_cursor = await request.app.state.storage.list_chats( + actor["id"], limit, cursor, + ) + except ValueError: + return error_response(400, "invalid_request", "bad cursor", rid) + return {"items": items, "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}", response_model=Chat) +async def get_chat( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + chat = await request.app.state.storage.get_chat(chat_id, actor["id"]) + if chat is None: + return error_response(404, "not_found", "chat not found", rid) + return chat + + +@router.post("/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: str, + data: CreateMessage, + request: Request, + authorization: str | None = Header(default=None), + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), +): + rid = request.state.request_id + validate_uuid(chat_id) + validate_idempotency_key(idempotency_key) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + + if not data.text.strip() and not data.attachment_ids: + return error_response(400, "invalid_request", "text or attachment_ids required", rid) + + try: + return await request.app.state.storage.add_message( + chat_id, actor["id"], data.text, + data.attachment_ids, idempotency_key, + ) + except ValueError as e: + if str(e) == "idempotency_conflict": + return error_response(409, "conflict", "idempotency key conflict", rid) + return error_response(404, "not_found", "chat not found", rid) + + +@router.get("/chats/{chat_id}/messages", response_model=MessagePage) +async def list_messages( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + items, next_cursor = await request.app.state.storage.list_messages( + chat_id, actor["id"], limit, cursor, + ) + except ValueError as e: + if str(e) == "bad_cursor": + return error_response(400, "invalid_request", "bad cursor", rid) + return error_response(404, "not_found", "chat not found", rid) + return {"items": items, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/api/health.py b/app/api/health.py new file mode 100644 index 0000000..537c05f --- /dev/null +++ b/app/api/health.py @@ -0,0 +1,53 @@ +import asyncio +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +router = APIRouter(tags=["health"]) + + +@router.get("/health/live") +async def liveness(): + return {"status": "ok"} + + +async def _check_postgres(request: Request) -> None: + async with request.app.state.storage._pool.acquire() as conn: + await conn.fetchval("SELECT 1") + + +async def _check_redis(request: Request) -> None: + store = request.app.state.session_store + if hasattr(store, "_client"): + await store._client.ping() + + +@router.get("/health/ready") +async def readiness(request: Request): + rid = getattr(request.state, "request_id", "") + try: + await asyncio.wait_for(_check_postgres(request), timeout=2.0) + except Exception: + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "db unavailable", + "request_id": rid, + } + }, + ) + try: + await asyncio.wait_for(_check_redis(request), timeout=2.0) + except Exception: + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "cache unavailable", + "request_id": rid, + } + }, + ) + return {"status": "ok"} \ No newline at end of file diff --git a/app/api/users.py b/app/api/users.py new file mode 100644 index 0000000..eabcf2e --- /dev/null +++ b/app/api/users.py @@ -0,0 +1,58 @@ +from fastapi import APIRouter, Header, HTTPException, Request +from fastapi.responses import JSONResponse + +from app.models import User, CreateUser +from app.security import hash_password, get_current_user + +router = APIRouter(tags=["users"]) + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +@router.post("/users", response_model=User, status_code=201) +async def register_user(data: CreateUser, request: Request): + storage = request.app.state.storage + rid = request.state.request_id + password_hash = hash_password(data.password) + try: + user = await storage.create_user( + data.username, data.display_name, password_hash, + ) + return user + except ValueError: + return error_response(409, "conflict", "username already taken", rid) + + +@router.get("/users/me", response_model=User) +async def current_user( + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + user, _ = await get_current_user(request, authorization) + except HTTPException as exc: + if exc.status_code == 503: + return error_response(503, "unavailable", "auth store unavailable", rid) + return error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user \ No newline at end of file diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 0000000..86058f3 --- /dev/null +++ b/app/cache.py @@ -0,0 +1,60 @@ +import json +from typing import Optional + +import redis.asyncio as redis + +from app.config import config + +CACHE_TTL_SECONDS = 30 + + +class Cache: + def __init__(self, client: redis.Redis) -> None: + self._client = client + + async def get_json(self, key: str) -> Optional[dict]: + raw = await self._client.get(key) + if raw is None: + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return None + + async def set_json(self, key: str, value: dict, ttl: int = CACHE_TTL_SECONDS) -> None: + await self._client.set(key, json.dumps(value), ex=ttl) + + async def delete(self, key: str) -> None: + await self._client.delete(key) + + async def invalidate_chat(self, chat_id: str) -> None: + """Удалить все ключи, связанные с чатом (история).""" + pattern = f"messages:{chat_id}:*" + cursor = 0 + while True: + cursor, keys = await self._client.scan(cursor=cursor, match=pattern, count=100) + if keys: + await self._client.delete(*keys) + if cursor == 0: + break + + async def try_lock(self, key: str, ttl: int = 5) -> bool: + """Атомарно установить ключ. True — если установили мы.""" + result = await self._client.set(key, "1", nx=True, ex=ttl) + return bool(result) + + async def unlock(self, key: str) -> None: + await self._client.delete(key) + + async def close(self) -> None: + await self._client.aclose() + + +def make_cache() -> Cache: + client = redis.from_url( + config.CACHE_URL, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + return Cache(client) \ No newline at end of file diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..73aa40e --- /dev/null +++ b/app/config.py @@ -0,0 +1,28 @@ +import os +import secrets + + +class Config: + DATABASE_URL: str = os.getenv( + "DATABASE_URL", + "postgresql://messenger:messenger@localhost:5432/messenger", + ) + SESSION_TTL_SECONDS: int = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + APP_HOST: str = os.getenv("APP_HOST", "0.0.0.0") + APP_PORT: int = int(os.getenv("APP_PORT", "8080")) + SESSION_STORE: str = os.getenv("SESSION_STORE", "postgres") + CACHE_URL: str = os.getenv( + "CACHE_URL", + "redis://valkey:6379/0", + ) + INSTANCE_ID: str = os.getenv("INSTANCE_ID") or secrets.token_hex(6) + + S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "minio:9000") + S3_PUBLIC_ENDPOINT: str = os.getenv("S3_PUBLIC_ENDPOINT", "localhost:9000") + S3_ACCESS_KEY: str = os.getenv("S3_ACCESS_KEY", "minioadmin") + S3_SECRET_KEY: str = os.getenv("S3_SECRET_KEY", "minioadmin") + S3_BUCKET: str = os.getenv("S3_BUCKET", "attachments") + S3_SECURE: bool = os.getenv("S3_SECURE", "false").lower() == "true" + + +config = Config() \ No newline at end of file diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..f49a598 --- /dev/null +++ b/app/db.py @@ -0,0 +1,30 @@ +import asyncpg +from app.config import config + + +class Database: + def __init__(self) -> None: + self._pool: asyncpg.Pool | None = None + + async def connect(self) -> None: + self._pool = await asyncpg.create_pool( + config.DATABASE_URL, + min_size=2, + max_size=10, + command_timeout=5, + timeout=5, + ) + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + self._pool = None + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("database is not connected") + return self._pool + + +db = Database() \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..4370c22 --- /dev/null +++ b/app/main.py @@ -0,0 +1,126 @@ +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError + +from app.db import db +from app.storage import Storage +from app.session_store import make_session_store +from app.cache import make_cache +from app.config import config +from app.s3 import s3 +from app.api import health, users, auth, chats, attachments + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await db.connect() + app.state.cache = make_cache() + app.state.storage = Storage(db.pool, app.state.cache) + app.state.session_store = make_session_store(db.pool) + await app.state.session_store.delete_expired() + s3.ensure_bucket() + app.state.s3 = s3 + yield + await app.state.cache.close() + await app.state.session_store.close() + await db.close() + + +app = FastAPI(title="MTUSI Messenger", version="1.4.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = rid + response = await call_next(request) + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=400, + content={"error": {"code": "invalid_request", "message": "validation error", "request_id": rid}}, + headers={"X-Request-Id": rid}, + ) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + headers = {"X-Request-Id": rid} + if exc.headers: + headers.update(exc.headers) + code_map = { + 400: "invalid_request", 401: "unauthorized", 404: "not_found", + 409: "conflict", 413: "payload_too_large", 415: "unsupported_media_type", + 429: "rate_limited", 503: "unavailable", + } + code = code_map.get(exc.status_code, "internal_error") + message = exc.detail if isinstance(exc.detail, str) else "error" + return JSONResponse( + status_code=exc.status_code, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers=headers, + ) + + +try: + from asyncpg.exceptions import ( + PostgresConnectionError, TooManyConnectionsError, + CannotConnectNowError, ConnectionDoesNotExistError, + ) + + @app.exception_handler(PostgresConnectionError) + @app.exception_handler(TooManyConnectionsError) + @app.exception_handler(CannotConnectNowError) + @app.exception_handler(ConnectionDoesNotExistError) + async def postgres_unavailable_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=503, + content={"error": {"code": "unavailable", "message": "database unavailable", "request_id": rid}}, + headers={"X-Request-Id": rid}, + ) +except ImportError: + pass + + +try: + import redis.exceptions as redis_exceptions + + @app.exception_handler(redis_exceptions.ConnectionError) + @app.exception_handler(redis_exceptions.TimeoutError) + async def redis_unavailable_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=503, + content={"error": {"code": "unavailable", "message": "cache unavailable", "request_id": rid}}, + headers={"X-Request-Id": rid}, + ) +except ImportError: + pass + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=500, + content={"error": {"code": "internal_error", "message": "internal error", "request_id": rid}}, + headers={"X-Request-Id": rid}, + ) + + +app.include_router(health.router) +app.include_router(users.router, prefix="/api/v1") +app.include_router(auth.router, prefix="/api/v1") +app.include_router(chats.router, prefix="/api/v1") +app.include_router(attachments.router, prefix="/api/v1") \ No newline at end of file diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 0000000..d368468 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,19 @@ +import asyncio +import sys + +from app.db import db +from app.migrations import apply_migrations + + +async def main() -> int: + await db.connect() + try: + await apply_migrations(db.pool) + finally: + await db.close() + print("migrations applied", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) \ No newline at end of file diff --git a/app/migrations.py b/app/migrations.py new file mode 100644 index 0000000..fea0eaf --- /dev/null +++ b/app/migrations.py @@ -0,0 +1,31 @@ +from pathlib import Path +import asyncpg + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS applied_migrations ( + name TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = { + row["name"] + for row in await conn.fetch("SELECT name FROM applied_migrations") + } + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO applied_migrations (name) VALUES ($1)", + path.name, + ) + print(f"applied migration: {path.name}", flush=True) \ No newline at end of file diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..36f20f7 --- /dev/null +++ b/app/models.py @@ -0,0 +1,111 @@ +from pydantic import BaseModel, Field, ConfigDict, field_validator +from typing import List, Optional + + +def _not_blank(value: str) -> str: + if not value.strip(): + raise ValueError("must contain non-whitespace characters") + return value + + +def _not_blank_or_empty(value: str) -> str: + if value and not value.strip(): + raise ValueError("must contain non-whitespace characters") + return value + + +class User(BaseModel): + id: str + username: str + display_name: str + created_at: str + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + _check_display_name = field_validator("display_name")(_not_blank) + + +class Chat(BaseModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + + _check_title = field_validator("title")(_not_blank) + + +class Message(BaseModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: List[str] + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field("", max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + _check_text = field_validator("text")(_not_blank_or_empty) + + +class Attachment(BaseModel): + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: List[dict] + error_code: Optional[str] + created_at: str + updated_at: str + + +class ChatPage(BaseModel): + items: List[Chat] + next_cursor: Optional[str] = None + + +class MessagePage(BaseModel): + items: List[Message] + next_cursor: Optional[str] = None + + +class Health(BaseModel): + status: str = "ok" + + +class Session(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: User + + +class ErrorBody(BaseModel): + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + error: ErrorBody \ No newline at end of file diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 0000000..789cd88 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,53 @@ +from datetime import timedelta +from io import BytesIO + +from minio import Minio + +from app.config import config + + +class S3Storage: + def __init__(self) -> None: + common = dict( + access_key=config.S3_ACCESS_KEY, + secret_key=config.S3_SECRET_KEY, + secure=config.S3_SECURE, + region="us-east-1", + ) + self._client = Minio(config.S3_ENDPOINT, **common) + self._presign_client = Minio(config.S3_PUBLIC_ENDPOINT, **common) + self._bucket = config.S3_BUCKET + + @property + def client(self) -> Minio: + return self._client + + @property + def bucket(self) -> str: + return self._bucket + + def ensure_bucket(self) -> None: + if not self._client.bucket_exists(self._bucket): + self._client.make_bucket(self._bucket) + + def put(self, key: str, data: bytes, content_type: str) -> None: + self._client.put_object( + self._bucket, key, BytesIO(data), len(data), content_type=content_type + ) + + def get(self, key: str): + return self._client.get_object(self._bucket, key) + + def stat(self, key: str): + return self._client.stat_object(self._bucket, key) + + def remove(self, key: str) -> None: + self._client.remove_object(self._bucket, key) + + def presigned_put(self, key: str, expires_seconds: int = 300) -> str: + return self._presign_client.presigned_put_object( + self._bucket, key, expires=timedelta(seconds=expires_seconds) + ) + + +s3 = S3Storage() \ No newline at end of file diff --git a/app/security.py b/app/security.py new file mode 100644 index 0000000..827ace2 --- /dev/null +++ b/app/security.py @@ -0,0 +1,83 @@ +import base64 +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError, InvalidHash +from fastapi import Header, HTTPException, Request + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + _hasher.verify(password_hash, password) + return True + except (VerifyMismatchError, VerificationError, InvalidHash): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(48) + + +def parse_basic(header: str | None) -> tuple[str, str]: + if not header or not header.lower().startswith("basic "): + raise HTTPException(status_code=401, detail="unauthorized") + encoded = header.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(encoded).decode("utf-8") + except Exception: + raise HTTPException(status_code=401, detail="unauthorized") + if ":" not in decoded: + raise HTTPException(status_code=401, detail="unauthorized") + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(header: str | None) -> str: + if not header or not header.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="unauthorized") + return header.split(" ", 1)[1].strip() + + +async def get_current_user( + request: Request, + authorization: str | None = Header(default=None), +): + token = parse_bearer(authorization) + session_store = request.app.state.session_store + storage = request.app.state.storage + + try: + session = await session_store.get(token) + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + + if session is None: + raise HTTPException(status_code=401, detail="unauthorized") + + expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00")) + if expires_at <= datetime.now(timezone.utc): + try: + await session_store.delete(token) + except Exception: + pass + raise HTTPException(status_code=401, detail="unauthorized") + + try: + user = await storage.get_user(session["user_id"]) + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + + if user is None: + raise HTTPException(status_code=401, detail="unauthorized") + return user, token + + +def session_expires_at(ttl_seconds: int) -> datetime: + return datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds) \ No newline at end of file diff --git a/app/session_store.py b/app/session_store.py new file mode 100644 index 0000000..6fa68cf --- /dev/null +++ b/app/session_store.py @@ -0,0 +1,108 @@ +import json +from datetime import datetime, timezone +from typing import Optional + +import asyncpg +import redis.asyncio as redis + +from app.config import config + + +def _now_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class PostgresSessionStore: + def __init__(self, pool: asyncpg.Pool) -> None: + self._pool = pool + + async def create(self, token: str, user_id: str, expires_at: datetime) -> dict: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO sessions (token, user_id, expires_at) + VALUES ($1, $2, $3) + RETURNING token, user_id, created_at, expires_at + """, + token, user_id, expires_at, + ) + return { + "token": row["token"], + "user_id": str(row["user_id"]), + "created_at": _now_iso(row["created_at"]), + "expires_at": _now_iso(row["expires_at"]), + } + + async def get(self, token: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT token, user_id, created_at, expires_at + FROM sessions WHERE token = $1 + """, + token, + ) + if row is None: + return None + return { + "token": row["token"], + "user_id": str(row["user_id"]), + "created_at": _now_iso(row["created_at"]), + "expires_at": _now_iso(row["expires_at"]), + } + + async def delete(self, token: str) -> None: + async with self._pool.acquire() as conn: + await conn.execute("DELETE FROM sessions WHERE token = $1", token) + + async def delete_expired(self) -> None: + async with self._pool.acquire() as conn: + await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + + async def close(self) -> None: + pass + +class RedisSessionStore: + def __init__(self, client: redis.Redis) -> None: + self._client = client + + async def create(self, token: str, user_id: str, expires_at: datetime) -> dict: + now = datetime.now(timezone.utc) + ttl = max(1, int((expires_at - now).total_seconds())) + payload = { + "token": token, + "user_id": user_id, + "created_at": _now_iso(now), + "expires_at": _now_iso(expires_at), + } + await self._client.set(f"session:{token}", json.dumps(payload), ex=ttl) + return payload + + async def get(self, token: str) -> Optional[dict]: + raw = await self._client.get(f"session:{token}") + if raw is None: + return None + return json.loads(raw) + + async def delete(self, token: str) -> None: + await self._client.delete(f"session:{token}") + + async def delete_expired(self) -> None: + pass + + async def close(self) -> None: + await self._client.aclose() + + +def make_session_store(pool: asyncpg.Pool) -> PostgresSessionStore | RedisSessionStore: + if config.SESSION_STORE == "redis": + client = redis.from_url( + config.CACHE_URL, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + return RedisSessionStore(client) + return PostgresSessionStore(pool) \ No newline at end of file diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..5f0fff3 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,509 @@ +import asyncio +import hashlib +import json +import uuid +from datetime import datetime, timezone +from typing import Optional + +import asyncpg + +from app.cache import Cache + + +def _now_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _canonical_hash(payload: dict) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +class Storage: + def __init__(self, pool: asyncpg.Pool, cache: Cache) -> None: + self._pool = pool + self._cache = cache + + async def create_user( + self, username: str, display_name: str, password_hash: str, + ) -> dict: + user_id = str(uuid.uuid4()) + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at + """, + user_id, username, display_name, password_hash, + ) + except asyncpg.UniqueViolationError: + raise ValueError("conflict") + return self._user_row(row) + + async def get_user(self, user_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, username, display_name, created_at FROM users WHERE id = $1", + user_id, + ) + return self._user_row(row) if row else None + + async def get_user_by_username(self, username: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = $1 + """, + username, + ) + return dict(row) if row else None + + async def create_chat( + self, title: str, member_ids: list[str], creator_id: str, + ) -> dict: + if creator_id not in member_ids: + raise ValueError("creator_not_member") + + chat_id = str(uuid.uuid4()) + async with self._pool.acquire() as conn: + async with conn.transaction(): + existing = await conn.fetchval( + "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", + member_ids, + ) + if existing != len(member_ids): + raise ValueError("invalid_member") + await conn.execute( + "INSERT INTO chats (id, title) VALUES ($1, $2)", + chat_id, title, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, uid) for uid in member_ids], + ) + created = await conn.fetchval( + "SELECT created_at FROM chats WHERE id = $1", chat_id, + ) + return { + "id": chat_id, + "title": title, + "member_ids": member_ids, + "created_at": _now_iso(created), + } + + async def get_chat(self, chat_id: str, actor_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, title, created_at FROM chats WHERE id = $1", + chat_id, + ) + if row is None: + return None + members = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", + chat_id, + ) + member_ids = [str(m["user_id"]) for m in members] + if actor_id not in member_ids: + return None + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": _now_iso(row["created_at"]), + } + + async def list_chats( + self, actor_id: str, limit: int, cursor: Optional[str], + ) -> tuple[list[dict], Optional[str]]: + async with self._pool.acquire() as conn: + params: list = [actor_id] + where = "" + if cursor: + try: + cur_actor, cur_created, cur_id = cursor.split("|", 2) + except ValueError: + raise ValueError("bad_cursor") + if cur_actor != actor_id: + raise ValueError("bad_cursor") + where = "AND (c.created_at, c.id) > ($2, $3)" + params += [ + datetime.fromisoformat(cur_created.replace("Z", "+00:00")), + cur_id, + ] + rows = await conn.fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = $1 {where} + ORDER BY c.created_at, c.id + LIMIT ${len(params) + 1} + """, + *params, limit + 1, + ) + items = [] + for row in rows[:limit]: + members = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", + row["id"], + ) + items.append({ + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m["user_id"]) for m in members], + "created_at": _now_iso(row["created_at"]), + }) + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = f"{actor_id}|{_now_iso(last['created_at'])}|{last['id']}" + return items, next_cursor + + async def is_chat_member(self, chat_id: str, user_id: str) -> bool: + async with self._pool.acquire() as conn: + return await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2", + chat_id, user_id, + ) is not None + + async def create_attachment( + self, att_id: str, chat_id: str, owner_id: str, object_key: str, + filename: str, content_type: str, size_bytes: int, sha256: str, + ) -> dict: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO attachments + (id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, status) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'ready') + RETURNING id, chat_id, owner_id, object_key, filename, + content_type, size_bytes, sha256, status, + error_code, created_at, updated_at + """, + att_id, chat_id, owner_id, object_key, + filename, content_type, size_bytes, sha256, + ) + return self._attachment_row(row, public=True) + + async def create_pending_attachment( + self, att_id: str, chat_id: str, owner_id: str, object_key: str, + filename: str, content_type: str, size_bytes: int, + ) -> dict: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO attachments + (id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, status) + VALUES ($1,$2,$3,$4,$5,$6,$7,'','pending') + RETURNING id, chat_id, owner_id, object_key, filename, + content_type, size_bytes, sha256, status, + error_code, created_at, updated_at + """, + att_id, chat_id, owner_id, object_key, + filename, content_type, size_bytes, + ) + return self._attachment_row(row, public=True) + + async def complete_attachment( + self, att_id: str, sha256: str, size_bytes: int, + ) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + UPDATE attachments + SET status='ready', sha256=$2, size_bytes=$3, updated_at=now() + WHERE id=$1 AND status='pending' + RETURNING id, chat_id, owner_id, object_key, filename, + content_type, size_bytes, sha256, status, + error_code, created_at, updated_at + """, + att_id, sha256, size_bytes, + ) + if row is None: + return None + return self._attachment_row(row, public=True) + + async def get_attachment(self, att_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT id, chat_id, owner_id, object_key, filename, + content_type, size_bytes, sha256, status, + error_code, created_at, updated_at + FROM attachments WHERE id=$1 + """, + att_id, + ) + if row is None: + return None + return self._attachment_row(row, public=False) + + async def _list_attachment_ids_conn(self, conn, message_id: str) -> list[str]: + rows = await conn.fetch( + "SELECT attachment_id FROM message_attachments WHERE message_id=$1", + message_id, + ) + return [str(r["attachment_id"]) for r in rows] + + async def add_message( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: Optional[list[str]] = None, + idempotency_key: Optional[str] = None, + ) -> dict: + attachment_ids = attachment_ids or [] + request_hash = _canonical_hash( + {"text": text, "attachment_ids": sorted(attachment_ids)} + ) + path = f"/api/v1/chats/{chat_id}/messages" + method = "POST" + + if idempotency_key is not None: + cached = await self._get_idempotent( + sender_id, method, path, idempotency_key, request_hash, + ) + if cached is not None: + return cached + + try: + async with self._pool.acquire() as conn: + async with conn.transaction(): + if attachment_ids: + valid = await conn.fetchval( + """ + SELECT count(*) FROM attachments + WHERE id = ANY($1::uuid[]) + AND chat_id = $2 + AND status = 'ready' + """, + attachment_ids, chat_id, + ) + if valid != len(attachment_ids): + raise ValueError("not_found") + + message_id = str(uuid.uuid4()) + row = await conn.fetchrow( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT $1, $2, $3, $4 + WHERE EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = $2 AND user_id = $3 + ) + RETURNING id, chat_id, sender_id, text, created_at + """, + message_id, chat_id, sender_id, text, + ) + if row is None: + raise ValueError("not_found") + + if attachment_ids: + await conn.executemany( + """ + INSERT INTO message_attachments (message_id, attachment_id) + VALUES ($1, $2) + """, + [(message_id, aid) for aid in attachment_ids], + ) + + message = self._message_row(row, list(attachment_ids)) + + await conn.execute( + "UPDATE chats SET version = version + 1 WHERE id = $1", + chat_id, + ) + + if idempotency_key is not None: + await conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, path, key, request_hash, + response_status, response_body) + VALUES ($1, $2, $3, $4, $5, $6, $7) + """, + sender_id, method, path, idempotency_key, + request_hash, 201, + json.dumps(message, ensure_ascii=False), + ) + except asyncpg.UniqueViolationError: + cached = await self._get_idempotent( + sender_id, method, path, idempotency_key, request_hash, + ) + if cached is not None: + return cached + raise + + await self._cache.invalidate_chat(chat_id) + return message + + async def _get_idempotent( + self, + user_id: str, + method: str, + path: str, + key: Optional[str], + request_hash: str, + ) -> Optional[dict]: + if key is None: + return None + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT request_hash, response_body + FROM idempotency_keys + WHERE user_id = $1 AND method = $2 AND path = $3 AND key = $4 + """, + user_id, method, path, key, + ) + if row is None: + return None + if row["request_hash"] != request_hash: + raise ValueError("idempotency_conflict") + return json.loads(row["response_body"]) + + async def list_messages( + self, chat_id: str, actor_id: str, limit: int, cursor: Optional[str], + ) -> tuple[list[dict], Optional[str]]: + async with self._pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + chat_id, actor_id, + ) + if is_member is None: + raise ValueError("not_found") + + cache_key = f"messages:{chat_id}:{limit}:{cursor or ''}" + + cached = await self._cache.get_json(cache_key) + if cached is not None: + version_now = await self._get_chat_version(chat_id) + if cached.get("version") == version_now: + return cached["items"], cached["next_cursor"] + await self._cache.delete(cache_key) + + lock_key = f"lock:{cache_key}" + got_lock = await self._cache.try_lock(lock_key, ttl=10) + + if not got_lock: + for _ in range(50): + await asyncio.sleep(0.1) + cached = await self._cache.get_json(cache_key) + if cached is not None: + version_now = await self._get_chat_version(chat_id) + if cached.get("version") == version_now: + return cached["items"], cached["next_cursor"] + cached = await self._cache.get_json(cache_key) + if cached is not None: + version_now = await self._get_chat_version(chat_id) + if cached.get("version") == version_now: + return cached["items"], cached["next_cursor"] + + try: + version_before = await self._get_chat_version(chat_id) + + async with self._pool.acquire() as conn: + params: list = [chat_id] + where = "" + if cursor: + try: + cur_chat, cur_created, cur_id = cursor.split("|", 2) + except ValueError: + raise ValueError("bad_cursor") + if cur_chat != chat_id: + raise ValueError("bad_cursor") + where = "AND (created_at, id) > ($2, $3)" + params += [ + datetime.fromisoformat(cur_created.replace("Z", "+00:00")), + cur_id, + ] + rows = await conn.fetch( + f""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 {where} + ORDER BY created_at, id + LIMIT ${len(params) + 1} + """, + *params, limit + 1, + ) + items = [] + for row in rows[:limit]: + att_ids = await self._list_attachment_ids_conn(conn, row["id"]) + items.append(self._message_row(row, att_ids)) + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = f"{chat_id}|{_now_iso(last['created_at'])}|{last['id']}" + + version_after = await self._get_chat_version(chat_id) + if got_lock and version_before == version_after: + await self._cache.set_json( + cache_key, + { + "items": items, + "next_cursor": next_cursor, + "version": version_after, + }, + ) + return items, next_cursor + finally: + if got_lock: + await self._cache.unlock(lock_key) + + async def _get_chat_version(self, chat_id: str) -> int: + async with self._pool.acquire() as conn: + row = await conn.fetchval( + "SELECT version FROM chats WHERE id = $1", + chat_id, + ) + return int(row) if row is not None else 0 + + @staticmethod + def _user_row(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": _now_iso(row["created_at"]), + } + + @staticmethod + def _message_row(row, attachment_ids: Optional[list[str]] = None) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": _now_iso(row["created_at"]), + "attachment_ids": attachment_ids or [], + } + + @staticmethod + def _attachment_row(row, public: bool) -> dict: + result = { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": _now_iso(row["created_at"]), + "updated_at": _now_iso(row["updated_at"]), + } + if not public: + result["_object_key"] = row["object_key"] + return result \ No newline at end of file diff --git a/compose-scale.yaml b/compose-scale.yaml new file mode 100644 index 0000000..4bf9532 --- /dev/null +++ b/compose-scale.yaml @@ -0,0 +1,7 @@ +services: + api: + deploy: + resources: + limits: + cpus: "${API_CPUS:-2.0}" + memory: ${API_MEMORY:-512M} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..9ef8462 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,188 @@ +services: + postgres: + image: postgres:16.4-alpine + command: + - "postgres" + - "-c" + - "shared_preload_libraries=pg_stat_statements" + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger + POSTGRES_DB: messenger + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + valkey: + image: valkey/valkey:8.0-alpine + command: ["valkey-server", "--appendonly", "yes", "--save", ""] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + minio: + image: bitnamilegacy/minio:2025.4.22-debian-12-r1 + user: "0:0" + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + volumes: + - minio_data:/data + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + minio-init: + image: bitnamilegacy/minio:2025.4.22-debian-12-r1 + user: "0:0" + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD && + mc mb --ignore-existing local/attachments && + mc anonymous set none local/attachments && + (mc admin user add local api-user api-user-secret-123 || true) && + (mc admin policy attach local readwrite --user api-user || true) && + echo 'bucket ready' + " + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + networks: + - messenger + + migrate: + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: redis + command: ["python", "-m", "app.migrate"] + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + api: + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: redis + SESSION_TTL_SECONDS: "3600" + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + S3_ENDPOINT: "minio:9000" + S3_PUBLIC_ENDPOINT: "localhost:9000" + S3_ACCESS_KEY: "api-user" + S3_SECRET_KEY: "api-user-secret-123" + S3_BUCKET: "attachments" + S3_SECURE: "false" + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + healthcheck: + test: [ "CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)" ] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + networks: + - messenger + deploy: + replicas: 2 + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + lb: + image: nginx:1.27-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + - api + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + reservations: + cpus: "0.1" + memory: 32M + +volumes: + postgres_data: + valkey_data: + minio_data: + +networks: + messenger: + driver: bridge \ No newline at end of file diff --git a/migrations/001_users.sql b/migrations/001_users.sql new file mode 100644 index 0000000..4e60309 --- /dev/null +++ b/migrations/001_users.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/002_chats.sql b/migrations/002_chats.sql new file mode 100644 index 0000000..e46d22e --- /dev/null +++ b/migrations/002_chats.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/003_chat_members.sql b/migrations/003_chat_members.sql new file mode 100644 index 0000000..cbed26e --- /dev/null +++ b/migrations/003_chat_members.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); \ No newline at end of file diff --git a/migrations/004_messages.sql b/migrations/004_messages.sql new file mode 100644 index 0000000..31b695d --- /dev/null +++ b/migrations/004_messages.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/005_sessions.sql b/migrations/005_sessions.sql new file mode 100644 index 0000000..ab1b4b1 --- /dev/null +++ b/migrations/005_sessions.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL +); \ No newline at end of file diff --git a/migrations/006_messages_index.sql b/migrations/006_messages_index.sql new file mode 100644 index 0000000..780d470 --- /dev/null +++ b/migrations/006_messages_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_messages_chat_created + ON messages(chat_id, created_at, id); \ No newline at end of file diff --git a/migrations/007_idempotency.sql b/migrations/007_idempotency.sql new file mode 100644 index 0000000..95e6cad --- /dev/null +++ b/migrations/007_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_status INTEGER NOT NULL, + response_body TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); \ No newline at end of file diff --git a/migrations/008_idempotency_index.sql b/migrations/008_idempotency_index.sql new file mode 100644 index 0000000..bb80795 --- /dev/null +++ b/migrations/008_idempotency_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_keys(created_at); \ No newline at end of file diff --git a/migrations/009_chat_version.sql b/migrations/009_chat_version.sql new file mode 100644 index 0000000..d46308b --- /dev/null +++ b/migrations/009_chat_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN IF NOT EXISTS version BIGINT NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/migrations/010_attachments.sql b/migrations/010_attachments.sql new file mode 100644 index 0000000..b708137 --- /dev/null +++ b/migrations/010_attachments.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id), + object_key TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'ready', + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id); \ No newline at end of file diff --git a/migrations/011_message_attachments.sql b/migrations/011_message_attachments.sql new file mode 100644 index 0000000..cbf805e --- /dev/null +++ b/migrations/011_message_attachments.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + PRIMARY KEY (message_id, attachment_id) +); \ No newline at end of file diff --git a/migrations/012_attachment_status.sql b/migrations/012_attachment_status.sql new file mode 100644 index 0000000..ae8627d --- /dev/null +++ b/migrations/012_attachment_status.sql @@ -0,0 +1,4 @@ +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check; +ALTER TABLE attachments + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('pending', 'ready')); \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..2c6b48d --- /dev/null +++ b/nginx.conf @@ -0,0 +1,31 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + upstream api_backend { + server api:8080 max_fails=1 fail_timeout=10s; + } + + server { + listen 8080; + server_name _; + + client_max_body_size 11m; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 1s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + proxy_next_upstream error timeout http_502 http_503 http_504; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ee42719 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +pydantic==2.11.4 +httpx==0.28.1 +pytest==8.3.5 +pytest-asyncio==0.25.3 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +redis==5.1.1 +minio==7.2.15 +Pillow==11.2.1 \ No newline at end of file diff --git a/scripts/check-balance.py b/scripts/check-balance.py new file mode 100644 index 0000000..4092857 --- /dev/null +++ b/scripts/check-balance.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import threading +import time +import uuid +from collections import Counter + +import httpx + +BASE = "http://localhost:8080" + + +def main(): + counter = Counter() + lock = threading.Lock() + + def worker(): + with httpx.Client(timeout=5) as c: + for _ in range(20): + try: + r = c.get(f"{BASE}/health/live") + if r.status_code == 200: + with lock: + counter[r.headers.get("x-instance-id")] += 1 + except Exception: + pass + time.sleep(0.05) + + threads = [threading.Thread(target=worker) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + print(counter) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/check-chat-validation.py b/scripts/check-chat-validation.py new file mode 100644 index 0000000..a5459a1 --- /dev/null +++ b/scripts/check-chat-validation.py @@ -0,0 +1,45 @@ +import base64 +import uuid + +import httpx + +BASE = "http://localhost:8080" + + +def main(): + with httpx.Client(timeout=10) as c: + username = "cv_" + uuid.uuid4().hex[:8] + password = "Test-password-1234" + c.post(f"{BASE}/api/v1/users", json={ + "username": username, "display_name": "C", "password": password, + }) + credential = base64.b64encode((username + ":" + password).encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}).json() + alice_id = s["user"]["id"] + alice_token = s["token"] + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "dup", "member_ids": [alice_id, alice_id]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("duplicate members:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "bad uuid", "member_ids": [alice_id, "not-a-uuid"]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("invalid uuid:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "ok", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("valid chat:", r.status_code) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/check-models.py b/scripts/check-models.py new file mode 100644 index 0000000..e60836f --- /dev/null +++ b/scripts/check-models.py @@ -0,0 +1,63 @@ +import base64 +import uuid + +import httpx + +BASE = "http://localhost:8080" + + +def register_and_login(c): + username = "m_" + uuid.uuid4().hex[:8] + password = "Test-password-1234" + c.post(f"{BASE}/api/v1/users", json={ + "username": username, "display_name": "T", "password": password, + }) + credential = base64.b64encode((username + ":" + password).encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}).json() + return s["user"]["id"], s["token"] + + +def main(): + with httpx.Client(timeout=10) as c: + alice_id, alice_token = register_and_login(c) + chat = c.post(f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + alice_token}).json() + + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": " "}, + headers={ + "Authorization": "Bearer " + alice_token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + print("blank text:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "hi", "sender_id": "extra"}, + headers={ + "Authorization": "Bearer " + alice_token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + print("extra sender_id:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/users", + json={"username": " ", "display_name": "X", "password": "Test-password-1234"}, + ) + print("blank username:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": " ", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("blank chat title:", r.status_code, r.text[:200]) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/fair-scaling-benchmark.py b/scripts/fair-scaling-benchmark.py new file mode 100644 index 0000000..8576a61 --- /dev/null +++ b/scripts/fair-scaling-benchmark.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +import base64 +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "fair_scaling.json" + + +def register_and_login(client): + username = "fair_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "F", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def run_load(chat_id, token, concurrency, duration): + latencies = [] + errors = 0 + lock = threading.Lock() + + def worker(): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=15) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=10", + headers={"Authorization": "Bearer " + token}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "fair-" + uuid.uuid4().hex[:6]}, + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + threads = [threading.Thread(target=worker) for _ in range(concurrency)] + started = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + return { + "concurrency": concurrency, + "duration_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(len(latencies) / elapsed, 2) if elapsed else 0, + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def write_nginx(strategy): + if strategy == "least_conn": + upstream = """ upstream api_backend { + least_conn; + server api:8080 max_fails=1 fail_timeout=10s; + }""" + else: + upstream = """ upstream api_backend { + server api:8080 max_fails=1 fail_timeout=10s; + }""" + + content = f"""worker_processes auto; + +events {{ + worker_connections 1024; +}} + +http {{ +{upstream} + + server {{ + listen 8080; + server_name _; + + location / {{ + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 1s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + proxy_next_upstream error timeout http_502 http_503 http_504; + }} + }} +}} +""" + (ROOT / "nginx.conf").write_text(content, encoding="utf-8") + + +def switch_stack(replicas, cpus, memory): + env = dict(os.environ) + env["API_CPUS"] = cpus + env["API_MEMORY"] = memory + + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", "down"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", + "up", "-d", "--build", "--scale", f"api={replicas}"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + for _ in range(120): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + time.sleep(8) + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready") + + +def run_series(name, strategy, configs): + results = [] + write_nginx(strategy) + print(f"\n### Series {name} ({strategy})", flush=True) + for cfg in configs: + replicas = cfg["replicas"] + cpus = cfg["cpus"] + memory = cfg["memory"] + print(f"\n=== replicas={replicas}, cpus={cpus}, memory={memory} ===", flush=True) + switch_stack(replicas, cpus, memory) + + with httpx.Client(timeout=15) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "fair", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + for run in range(3): + print(f" run {run+1}/3...", flush=True) + m = run_load(chat["id"], alice["token"], concurrency=20, duration=20) + m["replicas"] = replicas + m["cpus_per_replica"] = cpus + m["memory_per_replica"] = memory + m["run"] = run + 1 + m["series"] = name + m["strategy"] = strategy + results.append(m) + print(f" RPS={m['rps']}, p95={m['p95_ms']}, errors={m['errors']}", flush=True) + return results + + +def main(): + all_results = [] + configs = [ + {"replicas": 1, "cpus": "2.0", "memory": "512M"}, + {"replicas": 2, "cpus": "1.0", "memory": "256M"}, + {"replicas": 4, "cpus": "0.5", "memory": "128M"}, + ] + + all_results.extend(run_series("A_least_conn", "least_conn", configs)) + all_results.extend(run_series("B_round_robin", "round_robin", configs)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(all_results, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"\nSaved {len(all_results)} records to {EVIDENCE}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/measure-detection.py b/scripts/measure-detection.py new file mode 100644 index 0000000..b660008 --- /dev/null +++ b/scripts/measure-detection.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +import base64 +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "detection.json" + +STOP_LABEL = "mtusi2026lab3-api-" + + +def register_and_login(client: httpx.Client) -> dict: + username = "det_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "D", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def find_container_by_instance(target: str) -> str: + names = subprocess.run( + ["docker", "ps", "--filter", f"name={STOP_LABEL}", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True, + ).stdout.split() + for name in names: + result = subprocess.run( + ["docker", "logs", name], + capture_output=True, text=True, check=False, + ) + combined = result.stdout + result.stderr + if f"INSTANCE_ID={target}" in combined: + return name + raise RuntimeError(f"no container with INSTANCE_ID={target}") + + +def get_instances(n: int = 30) -> set: + seen = set() + with httpx.Client(timeout=2) as c: + for _ in range(n): + try: + r = c.get(f"{BASE}/health/live") + if r.status_code == 200: + seen.add(r.headers.get("x-instance-id")) + except Exception: + pass + return seen + + +def main(): + print("[measure] preparing user and chat", flush=True) + with httpx.Client(timeout=5) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "det", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + instances_before = get_instances() + if len(instances_before) < 2: + print(f"[measure] need >=2 replicas, got {instances_before}", file=sys.stderr) + return 1 + print(f"[measure] instances before: {instances_before}", flush=True) + + target = sorted(instances_before)[0] + container = find_container_by_instance(target) + print(f"[measure] will stop {container} (instance={target})", flush=True) + + shared = { + "latencies": [], + "errors": 0, + "instances": [], + "stop_time": None, + "first_success_after_stop": None, + "first_success_instance": None, + "stop_seen": False, + } + lock = threading.Lock() + running = {"flag": True} + + def worker(): + with httpx.Client(timeout=3) as c: + while running["flag"]: + t0 = time.monotonic() + try: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "det-" + uuid.uuid4().hex[:8]}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + dt = time.monotonic() - t0 + if r.status_code == 201: + inst = r.headers.get("x-instance-id") + with lock: + shared["latencies"].append(dt) + shared["instances"].append(inst) + if shared["stop_time"] is not None and not shared["stop_seen"]: + if inst != target: + shared["first_success_after_stop"] = time.monotonic() - shared["stop_time"] + shared["first_success_instance"] = inst + shared["stop_seen"] = True + else: + with lock: + shared["errors"] += 1 + except Exception: + with lock: + shared["errors"] += 1 + + print("[measure] starting background load", flush=True) + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + + time.sleep(3) + with lock: + baseline_count = len(shared["latencies"]) + baseline_rps = baseline_count / 3.0 + print(f"[measure] baseline RPS ~{baseline_rps:.1f}", flush=True) + + print(f"[measure] stopping {container}", flush=True) + with lock: + shared["stop_time"] = time.monotonic() + subprocess.run(["docker", "stop", "--time=2", container], check=True) + + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + with lock: + if shared["first_success_after_stop"] is not None: + break + time.sleep(0.05) + + time.sleep(2) + + running["flag"] = False + for t in threads: + t.join(timeout=5) + + with lock: + report = { + "target_instance": target, + "target_container": container, + "instances_before": list(instances_before), + "total_successful": len(shared["latencies"]), + "total_errors": shared["errors"], + "baseline_rps_3s": round(baseline_rps, 2), + "detection_seconds": ( + round(shared["first_success_after_stop"], 3) + if shared["first_success_after_stop"] else None + ), + "first_success_instance_after_stop": shared["first_success_instance"], + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + existing = [] + if EVIDENCE.exists(): + try: + existing = json.loads(EVIDENCE.read_text(encoding="utf-8")) + except json.JSONDecodeError: + existing = [] + existing.append(report) + EVIDENCE.write_text( + json.dumps(existing, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + print(f"[measure] restoring {container}", flush=True) + subprocess.run( + ["docker", "compose", "--env-file", ".env", "up", "-d", "--scale", "api=2"], + cwd=ROOT, check=True, + ) + time.sleep(5) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/plot-scaling.py b/scripts/plot-scaling.py new file mode 100644 index 0000000..6fbd70b --- /dev/null +++ b/scripts/plot-scaling.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from pathlib import Path + +import plotly.graph_objects as go + +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" + + +def make_rps_chart(): + fig = go.Figure() + fig.add_trace(go.Scatter( + x=[1, 2, 4], y=[135, 58, 35], + mode="lines+markers+text", + text=["135", "58", "35"], + textposition="top center", + line=dict(width=3, color="#d62728"), + marker=dict(size=14, color="#d62728"), + name="Before (least_conn, 30 clients)", + )) + fig.add_trace(go.Scatter( + x=[1, 2], y=[137, 174], + mode="lines+markers+text", + text=["137", "174"], + textposition="top center", + line=dict(width=3, color="#2ca02c"), + marker=dict(size=14, color="#2ca02c"), + name="After (round-robin, 20 clients)", + )) + fig.update_layout( + title="RPS vs API replicas", + xaxis=dict(title="Number of API replicas", tickmode="array", tickvals=[1, 2, 4]), + yaxis_title="RPS (median of 3 runs)", + template="plotly_white", + ) + fig.write_image(str(EVIDENCE / "scaling-rps.png")) + + +def make_p95_chart(): + fig = go.Figure() + fig.add_trace(go.Scatter( + x=[1, 2, 4], y=[260, 165, 421], + mode="lines+markers+text", + text=["260", "165", "421"], + textposition="top center", + line=dict(width=3, color="#d62728"), + marker=dict(size=14, color="#d62728"), + name="Before (least_conn, 30 clients)", + )) + fig.add_trace(go.Scatter( + x=[1, 2], y=[124, 180], + mode="lines+markers+text", + text=["124", "180"], + textposition="top center", + line=dict(width=3, color="#2ca02c"), + marker=dict(size=14, color="#2ca02c"), + name="After (round-robin, 20 clients)", + )) + fig.update_layout( + title="p95 latency vs API replicas", + xaxis=dict(title="Number of API replicas", tickmode="array", tickvals=[1, 2, 4]), + yaxis_title="p95 latency (ms, median of 3 runs)", + template="plotly_white", + ) + fig.write_image(str(EVIDENCE / "scaling-p95.png")) + + +def main(): + make_rps_chart() + make_p95_chart() + print(f"Saved charts to {EVIDENCE}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/presign_compare.py b/scripts/presign_compare.py new file mode 100644 index 0000000..5ae7ec9 --- /dev/null +++ b/scripts/presign_compare.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Compare proxy vs presigned upload: API RAM peak and bytes through API.""" +import base64 +import io +import subprocess +import threading +import time +import uuid + +import httpx +from PIL import Image + +BASE = "http://localhost:8080" +SIZES = [1 * 1024 * 1024, 9 * 1024 * 1024] + + +def _png(size_bytes): + img = Image.new("RGB", (1000, 1000), (0, 0, 255)) + buf = io.BytesIO() + img.save(buf, format="PNG") + data = buf.getvalue() + if len(data) < size_bytes: + data = data + b"\x00" * (size_bytes - len(data)) + return data[:size_bytes] + + +def _user(c): + u = "cmp_" + uuid.uuid4().hex[:8] + p = "Test-password-1234" + c.post(f"{BASE}/api/v1/users", json={"username": u, "display_name": "C", "password": p}) + cred = base64.b64encode(f"{u}:{p}".encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}).json() + return {"id": s["user"]["id"], "token": s["token"]} + + +def _chat(c, a, b): + return c.post( + f"{BASE}/api/v1/chats", + json={"title": "cmp", "member_ids": [a["id"], b["id"]]}, + headers={"Authorization": "Bearer " + a["token"]}, + ).json() + + +def _mem(container): + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container], + capture_output=True, text=True, check=True, + ).stdout.strip() + v = out.split("/")[0].strip() + units = { + "B": 1, + "KiB": 1024, "Ki": 1024, + "MiB": 1024**2, "Mi": 1024**2, + "GiB": 1024**3, "Gi": 1024**3, + } + for u in sorted(units, key=len, reverse=True): + if v.endswith(u): + return int(float(v[: -len(u)]) * units[u]) + return 0 + + +def _measure(container, fn): + baseline = _mem(container) + peak = baseline + stop = {"flag": False} + + def sampler(): + nonlocal peak + while not stop["flag"]: + try: + m = _mem(container) + if m > peak: + peak = m + except Exception: + pass + time.sleep(0.15) + + t = threading.Thread(target=sampler) + t.start() + started = time.monotonic() + fn() + elapsed = time.monotonic() - started + stop["flag"] = True + t.join(timeout=5) + return baseline, peak, elapsed + + +def main(): + containers = subprocess.run( + ["docker", "ps", "--filter", "name=mtusi2026lab4-api-", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True, + ).stdout.split() + if not containers: + raise SystemExit("no api containers found") + target = containers[0] + + with httpx.Client(timeout=60) as c: + a = _user(c) + b = _user(c) + chat = _chat(c, a, b) + + for size in SIZES: + data = _png(size) + + def proxy_upload(): + with httpx.Client(timeout=60) as c: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments", + params={"filename": "x.png"}, + content=data, + headers={"Authorization": "Bearer " + a["token"], "Content-Type": "image/png"}, + ) + assert r.status_code == 201, r.text + + def presigned_upload(): + with httpx.Client(timeout=60) as c: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments/presign", + params={"filename": "x.png", "size": len(data), "content_type": "image/png"}, + headers={"Authorization": "Bearer " + a["token"]}, + ) + assert r.status_code == 201, r.text + p = r.json() + put = httpx.put(p["upload_url"], content=data, headers={"Content-Type": "image/png"}, timeout=60) + assert put.status_code == 200, put.text + r = c.post( + f"{BASE}/api/v1/attachments/{p['attachment_id']}/complete", + headers={"Authorization": "Bearer " + a["token"]}, + ) + assert r.status_code == 201, r.text + + for label, fn in (("proxy", proxy_upload), ("presigned", presigned_upload)): + baseline, peak, elapsed = _measure(target, fn) + print( + f"size={size} mode={label} " + f"elapsed={elapsed:.2f}s " + f"api_ram_baseline={baseline/1024**2:.1f}MiB " + f"api_ram_peak={peak/1024**2:.1f}MiB " + f"delta={(peak-baseline)/1024**2:.1f}MiB" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/ram_experiment.py b/scripts/ram_experiment.py new file mode 100644 index 0000000..8cc2ca3 --- /dev/null +++ b/scripts/ram_experiment.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Measure API RAM peak under >=10 concurrent uploads via docker stats.""" +import base64 +import io +import subprocess +import threading +import time +import uuid + +import httpx +from PIL import Image + +BASE = "http://localhost:8080" +CONCURRENCY = 10 +UPLOAD_BYTES = 9 * 1024 * 1024 + + +def _png(size_bytes: int) -> bytes: + img = Image.new("RGB", (1000, 1000), (255, 0, 0)) + buf = io.BytesIO() + img.save(buf, format="PNG") + data = buf.getvalue() + if len(data) < size_bytes: + data = data + b"\x00" * (size_bytes - len(data)) + return data[:size_bytes] + + +def _register_and_login(c): + u = "ram_" + uuid.uuid4().hex[:8] + p = "Test-password-1234" + r = c.post(f"{BASE}/api/v1/users", json={"username": u, "display_name": "R", "password": p}) + assert r.status_code == 201, r.text + user = r.json() + cred = base64.b64encode(f"{u}:{p}".encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}).json() + return {"id": user["id"], "token": s["token"]} + + +def _stats_mem(container: str) -> int: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container], + capture_output=True, text=True, check=True, + ).stdout.strip() + value = out.split("/")[0].strip() + units = { + "B": 1, + "KiB": 1024, "Ki": 1024, + "MiB": 1024**2, "Mi": 1024**2, + "GiB": 1024**3, "Gi": 1024**3, + } + for u in sorted(units, key=len, reverse=True): + if value.endswith(u): + return int(float(value[: -len(u)]) * units[u]) + raise ValueError(f"unexpected docker stats format: {out!r}") + + +def main(): + data = _png(UPLOAD_BYTES) + with httpx.Client(timeout=30) as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "ram", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + containers = subprocess.run( + ["docker", "ps", "--filter", "name=mtusi2026lab4-api-", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True, + ).stdout.split() + if not containers: + raise SystemExit("no api containers found") + target = containers[0] + baseline = _stats_mem(target) + + errors = [] + lock = threading.Lock() + + def worker(): + try: + with httpx.Client(timeout=60) as c: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments", + params={"filename": "ram.png"}, + content=data, + headers={ + "Authorization": "Bearer " + alice["token"], + "Content-Type": "image/png", + }, + ) + if r.status_code != 201: + with lock: + errors.append((r.status_code, r.text[:200])) + except Exception as e: + with lock: + errors.append((0, str(e))) + + threads = [threading.Thread(target=worker) for _ in range(CONCURRENCY)] + peak = baseline + stop = {"flag": False} + + def sampler(): + nonlocal peak + while not stop["flag"]: + try: + m = _stats_mem(target) + if m > peak: + peak = m + except Exception: + pass + time.sleep(0.2) + + sampler_thread = threading.Thread(target=sampler) + sampler_thread.start() + started = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + stop["flag"] = True + sampler_thread.join(timeout=5) + + print(f"container: {target}") + print(f"uploads: {CONCURRENCY} x {UPLOAD_BYTES} bytes") + print(f"elapsed: {elapsed:.2f}s") + print(f"errors: {len(errors)}") + for e in errors: + print(" error:", e) + print(f"baseline RAM: {baseline / 1024**2:.1f} MiB") + print(f"peak RAM: {peak / 1024**2:.1f} MiB") + print(f"delta: {(peak - baseline) / 1024**2:.1f} MiB") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 0000000..2e4eca3 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Find and clean orphan S3 objects older than GRACE; log metadata without object. + +Usage (inside api container): + docker compose exec -T api-1 python /tmp/reconcile.py --dry-run + docker compose exec -T api-1 python /tmp/reconcile.py +""" +import argparse +import asyncio +import os +from datetime import datetime, timedelta, timezone + +import asyncpg +from minio import Minio + +GRACE_HOURS = 1 +BUCKET = os.getenv("S3_BUCKET", "attachments") +DB_URL = os.getenv( + "DATABASE_URL", + "postgresql://messenger:messenger@postgres:5432/messenger", +) +S3 = Minio( + os.getenv("S3_ENDPOINT", "minio:9000"), + access_key=os.getenv("S3_ACCESS_KEY", "api-user"), + secret_key=os.getenv("S3_SECRET_KEY", "api-user-secret-123"), + secure=False, +) + + +async def main(dry_run: bool) -> int: + pool = await asyncpg.create_pool(DB_URL) + try: + async with pool.acquire() as conn: + rows = await conn.fetch("SELECT object_key FROM attachments") + known = {r["object_key"] for r in rows} + + cutoff = datetime.now(timezone.utc) - timedelta(hours=GRACE_HOURS) + removed = 0 + skipped_fresh = 0 + for obj in S3.list_objects(BUCKET, recursive=True): + if obj.object_name in known: + continue + if obj.last_modified and obj.last_modified > cutoff: + skipped_fresh += 1 + continue + print(f"orphan: {obj.object_name}") + if not dry_run: + S3.remove_object(BUCKET, obj.object_name) + removed += 1 + + missing = 0 + async with pool.acquire() as conn: + rows = await conn.fetch("SELECT id, object_key FROM attachments") + for r in rows: + try: + S3.stat_object(BUCKET, r["object_key"]) + except Exception: + missing += 1 + print(f"missing object for attachment {r['id']}: {r['object_key']}") + + mode = "dry-run" if dry_run else "apply" + print( + f"reconcile ({mode}): orphans_found={removed} " + f"fresh_skipped={skipped_fresh} metadata_without_object={missing}" + ) + return 0 + finally: + await pool.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + raise SystemExit(asyncio.run(main(args.dry_run))) \ No newline at end of file diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..09abb2c --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0restart.py" %* \ No newline at end of file diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 0000000..90814d5 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE_URL = "http://localhost:8080" + + +def run(cmd): + print(f"[restart] $ {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, cwd=ROOT) + if result.returncode != 0: + print( + f"[restart] command failed with code {result.returncode}", + file=sys.stderr, + ) + sys.exit(result.returncode) + + +def main(): + run(["docker", "compose", "--env-file", ".env", "down"]) + run([ + "docker", "compose", "--env-file", ".env", + "up", "-d", "--build", "--scale", "api=2", + ]) + + print("[restart] waiting for /health/ready", flush=True) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen( + f"{BASE_URL}/health/ready", timeout=2 + ) as r: + if r.status == 200: + print("[restart] api is ready", flush=True) + return 0 + except Exception: + time.sleep(1) + print( + "[restart] api did not become ready in 90 seconds", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..d5c1313 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "[restart.sh] docker compose down (volumes preserved)" +docker compose --env-file .env down + +echo "[restart.sh] docker compose up -d --build --scale api=2" +docker compose --env-file .env up -d --build --scale api=2 + +echo "[restart.sh] waiting for /health/ready" +for i in $(seq 1 90); do + if curl -fsS http://localhost:8080/health/ready >/dev/null 2>&1; then + echo "[restart.sh] api is ready" + exit 0 + fi + sleep 1 +done + +echo "[restart.sh] api did not become ready in 90 seconds" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/rolling-update.py b/scripts/rolling-update.py new file mode 100644 index 0000000..069321e --- /dev/null +++ b/scripts/rolling-update.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +import base64 +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "rolling-update.json" + + +def register_and_login(client): + username = "roll_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "R", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def main(): + with httpx.Client(timeout=5) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "roll", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + running = {"flag": True} + latencies = [] + errors = 0 + lock = threading.Lock() + + def worker(): + nonlocal errors + with httpx.Client(timeout=3) as c: + while running["flag"]: + t0 = time.monotonic() + try: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "roll-" + uuid.uuid4().hex[:8]}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + dt = time.monotonic() - t0 + if r.status_code == 201: + with lock: + latencies.append(dt) + else: + with lock: + errors += 1 + except Exception: + with lock: + errors += 1 + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + + time.sleep(3) + with lock: + before = len(latencies) + print(f"[roll] baseline RPS ~{before/3:.1f}", flush=True) + + for replica_name in ["mtusi2026lab3-api-1", "mtusi2026lab3-api-2"]: + print(f"[roll] restarting {replica_name}", flush=True) + subprocess.run(["docker", "restart", replica_name], check=True) + time.sleep(8) + + time.sleep(2) + + running["flag"] = False + for t in threads: + t.join(timeout=5) + + with lock: + report = { + "total_successful": len(latencies), + "total_errors": errors, + "baseline_rps_3s": round(before / 3.0, 2), + "avg_latency_ms": round(1000 * sum(latencies) / len(latencies), 2) if latencies else None, + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/scaling-benchmark.py b/scripts/scaling-benchmark.py new file mode 100644 index 0000000..7d23f65 --- /dev/null +++ b/scripts/scaling-benchmark.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +import base64 +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "scaling.json" + + +def register_and_login(client): + username = "scale_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "S", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def run_load(chat_id: str, token: str, concurrency: int, duration: float): + latencies = [] + errors = 0 + lock = threading.Lock() + + def worker(): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=15) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=10", + headers={"Authorization": "Bearer " + token}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "s-" + uuid.uuid4().hex[:6]}, + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + threads = [threading.Thread(target=worker) for _ in range(concurrency)] + started = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + return { + "concurrency": concurrency, + "duration_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(len(latencies) / elapsed, 2) if elapsed else 0, + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def switch_stack(replicas: int, cpus: str, memory: str): + env = dict(os.environ) + env["API_CPUS"] = cpus + env["API_MEMORY"] = memory + + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", "down"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", + "up", "-d", "--build", "--scale", f"api={replicas}"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + + for _ in range(120): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + time.sleep(8) + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready") + + +def experiment_series(name, configurations, concurrency): + results = [] + for cfg in configurations: + replicas = cfg["replicas"] + cpus = cfg["cpus"] + memory = cfg["memory"] + print(f"\n=== {name}: replicas={replicas}, cpus={cpus}, memory={memory}, clients={concurrency} ===", flush=True) + switch_stack(replicas, cpus, memory) + + with httpx.Client(timeout=15) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "scale", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + for run in range(3): + print(f" run {run+1}/3...", flush=True) + m = run_load(chat["id"], alice["token"], concurrency=concurrency, duration=20) + m["replicas"] = replicas + m["cpus_per_replica"] = cpus + m["memory_per_replica"] = memory + m["run"] = run + 1 + m["series"] = name + results.append(m) + print(f" RPS={m['rps']}, p95={m['p95_ms']}, errors={m['errors']}", flush=True) + + return results + + +def main(): + all_results = [] + + print("### Baseline: 1 replica, 2 CPU, 20 clients", flush=True) + all_results.extend(experiment_series( + "baseline", + [{"replicas": 1, "cpus": "2.0", "memory": "512M"}], + concurrency=20, + )) + + print("\n### Improved: 2 replicas, 1 CPU, 20 clients", flush=True) + all_results.extend(experiment_series( + "improved", + [{"replicas": 2, "cpus": "1.0", "memory": "256M"}], + concurrency=20, + )) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(all_results, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"\nSaved {len(all_results)} records to {EVIDENCE}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..ba5c965 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0stop-one.py" %* \ No newline at end of file diff --git a/scripts/stop-one.py b/scripts/stop-one.py new file mode 100644 index 0000000..e836501 --- /dev/null +++ b/scripts/stop-one.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import os +import subprocess +import sys + + +def main(): + target = os.environ.get("TARGET_INSTANCE_ID") + if not target: + print("[stop-one] TARGET_INSTANCE_ID not set", file=sys.stderr) + return 1 + + names = subprocess.run( + ["docker", "ps", "--filter", "name=mtusi2026lab4-api-", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True, + ).stdout.split() + + for container in names: + logs = subprocess.run( + ["docker", "logs", container], + capture_output=True, text=True, check=False, + ) + combined = logs.stdout + logs.stderr + if f"INSTANCE_ID={target}" in combined: + print(f"[stop-one] stopping {container} (instance={target})", flush=True) + subprocess.run(["docker", "stop", "--time=2", container], check=True) + return 0 + + print(f"[stop-one] no container found for INSTANCE_ID={target}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..5c084f9 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET="${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID not set}" + +for container in $(docker ps --filter "name=mtusi2026lab4-api-" --format "{{.Names}}"); do + if docker logs "$container" 2>&1 | grep -q "INSTANCE_ID=$TARGET"; then + echo "[stop-one] stopping $container (instance=$TARGET)" + docker stop "$container" + exit 0 + fi +done + +echo "[stop-one] no container found for INSTANCE_ID=$TARGET" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/test-idempotency.py b/scripts/test-idempotency.py new file mode 100644 index 0000000..cb5e2a5 --- /dev/null +++ b/scripts/test-idempotency.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import base64 +import json +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "idempotency.json" + + +def register_and_login(client): + username = "idem_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "I", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def main(): + with httpx.Client(timeout=10) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "idem", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + key = uuid.uuid4().hex + text = "idempotency-test-" + uuid.uuid4().hex[:8] + results = [] + lock = threading.Lock() + barrier = threading.Barrier(10) + + def worker(): + try: + with httpx.Client(timeout=10) as c: + barrier.wait() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": key, + }, + ) + with lock: + results.append((r.status_code, r.headers.get("x-instance-id"))) + except Exception as e: + with lock: + results.append((0, str(e))) + + threads = [threading.Thread(target=worker) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + + time.sleep(1) + + with httpx.Client(timeout=10) as c: + page = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=100", + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + matching = [m for m in page["items"] if m["text"] == text] + message_ids = {m["id"] for m in matching} + instances_seen = list({r[1] for r in results if r[1] and r[0] > 0}) + statuses = [r[0] for r in results] + + report = { + "concurrent_requests": len(results), + "statuses": statuses, + "unique_instances": instances_seen, + "messages_created": len(matching), + "unique_message_ids": len(message_ids), + "idempotency_ok": len(matching) == 1, + "instances_count": len(instances_seen), + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/student-tests/__init__.py b/student-tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/student-tests/benchmark.py b/student-tests/benchmark.py new file mode 100644 index 0000000..2b5e843 --- /dev/null +++ b/student-tests/benchmark.py @@ -0,0 +1,117 @@ +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +import httpx + +BASE = "http://localhost:8080" + + +def register(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + return client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "Bench"}, + ).json() + + +def setup(history_size: int): + with httpx.Client(timeout=120) as c: + alice = register(c) + bob = register(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "bench", "member_ids": [alice["id"], bob["id"]]}, + headers={"X-User-Id": alice["id"]}, + ).json() + for i in range(history_size): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"seed-{i}"}, + headers={"X-User-Id": alice["id"]}, + ) + return alice, bob, chat + + +def measure(chat_id: str, actor_id: str, concurrency: int, duration: float = 5.0): + latencies = [] + errors = 0 + lock = __import__("threading").Lock() + + def worker(worker_id): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=10) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=50", + headers={"X-User-Id": actor_id}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "load"}, + headers={"X-User-Id": actor_id}, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + t_start = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + list(pool.map(lambda w: worker(w), range(concurrency))) + elapsed = time.monotonic() - t_start + + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 3) + + rps = len(latencies) / elapsed if elapsed else 0 + return { + "concurrency": concurrency, + "elapsed_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(rps, 2), + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def main(): + results = [] + for history in (100, 1000, 10000): + print(f"Setup history={history}...") + alice, _, chat = setup(history) + for conc in (1, 10, 50): + print(f" measure history={history} concurrency={conc}...") + m = measure(chat["id"], alice["id"], conc, duration=5.0) + m["history_size"] = history + results.append(m) + print(" ", json.dumps(m, ensure_ascii=False)) + + with open("benchmark_results.json", "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) + print("Saved to benchmark_results.json") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/cache_experiment.py b/student-tests/cache_experiment.py new file mode 100644 index 0000000..83b6034 --- /dev/null +++ b/student-tests/cache_experiment.py @@ -0,0 +1,176 @@ +"""Cache stampede experiment. + +Measures SQL calls and latency for parallel GET /messages with cold cache. +Run twice: before and after single-flight protection. +""" +import base64 +import json +import os +import subprocess +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +EVIDENCE = Path(__file__).resolve().parent.parent / "evidence" / "cache_experiment.json" + + +def register_and_login(client: httpx.Client) -> dict: + username = "cache_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Cache", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def sql_calls_for_query(substring: str) -> int: + """Count SQL calls from pg_stat_statements matching substring.""" + sql = ( + "SELECT COALESCE(SUM(calls), 0) FROM pg_stat_statements " + f"WHERE query ILIKE '%{substring}%';" + ) + result = subprocess.run( + [ + "docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-t", "-A", "-c", sql, + ], + capture_output=True, text=True, check=True, + ) + return int(result.stdout.strip() or "0") + + +def reset_pg_stats() -> None: + subprocess.run( + [ + "docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-c", "SELECT pg_stat_statements_reset();", + ], + capture_output=True, text=True, check=True, + ) + + +def flush_cache() -> None: + subprocess.run( + [ + "docker", "compose", "exec", "-T", "valkey", + "sh", "-c", + "valkey-cli --scan --pattern 'messages:*' | xargs -r valkey-cli DEL", + ], + capture_output=True, text=True, check=True, + ) + +def run_parallel_gets(chat_id: str, token: str, concurrency: int = 50): + """Fire N parallel GET requests; return list of latencies and statuses.""" + results = [] + + def worker(): + with httpx.Client(timeout=15) as c: + t0 = time.monotonic() + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=50", + headers={"Authorization": "Bearer " + token}, + ) + dt = time.monotonic() - t0 + return dt, r.status_code + + with ThreadPoolExecutor(max_workers=concurrency) as pool: + results = list(pool.map(lambda _: worker(), range(concurrency))) + return results + + +def main(): + label = os.getenv("LABEL", "before") + concurrency = int(os.getenv("CONCURRENCY", "50")) + + with httpx.Client(timeout=30) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "cache", "member_ids": [alice["id"], bob["id"]]}, + headers=auth(alice), + ).json() + for i in range(100): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"seed-{i:03d}"}, + headers=auth(alice), + ) + + flush_cache() + reset_pg_stats() + time.sleep(0.5) + + print(f"Running {concurrency} parallel GETs (label={label})...") + started = time.monotonic() + results = run_parallel_gets(chat["id"], alice["token"], concurrency) + elapsed = time.monotonic() - started + + latencies = sorted(dt for dt, _ in results) + statuses = [s for _, s in results] + ok_count = sum(1 for s in statuses if s == 200) + errors = concurrency - ok_count + + sql_calls = sql_calls_for_query("FROM messages") + + def pct(p): + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + report = { + "label": label, + "concurrency": concurrency, + "elapsed_s": round(elapsed, 3), + "ok": ok_count, + "errors": errors, + "latency_ms": { + "p50": pct(0.5), + "p95": pct(0.95), + "p99": pct(0.99), + }, + "sql_calls_from_messages": sql_calls, + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + existing = [] + if EVIDENCE.exists(): + try: + existing = json.loads(EVIDENCE.read_text(encoding="utf-8")) + except json.JSONDecodeError: + existing = [] + existing.append(report) + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(existing, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"Saved to {EVIDENCE}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..9b3c9c4 --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,90 @@ +"""Populate a chat with 10000 messages and run EXPLAIN (ANALYZE, BUFFERS). + +Saves the plan to evidence/explain.txt for the report. +""" +import base64 +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +EVIDENCE = Path(__file__).resolve().parent.parent / "evidence" / "explain.txt" + + +def register_and_login(client: httpx.Client) -> dict: + username = "explain_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Explain", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return {"id": user["id"], "token": session["token"]} + + +def auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def main(): + total = 10000 + print(f"Setting up {total} messages...") + + with httpx.Client(timeout=120) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "explain", "member_ids": [alice["id"], bob["id"]]}, + headers=auth(alice), + ).json() + + print(f"chat_id = {chat['id']}") + + batch = 200 + started = time.monotonic() + for i in range(0, total, batch): + for j in range(batch): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i+j:05d}"}, + headers=auth(alice), + ) + if (i + batch) % 2000 == 0: + elapsed = time.monotonic() - started + print(f" {i+batch}/{total} messages, {elapsed:.1f}s") + + print(f"Setup done in {time.monotonic() - started:.1f}s") + print(f"chat_id: {chat['id']}") + + print("Run in psql:") + print(f""" +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '{chat['id']}' + AND (created_at, id) > ('2026-01-01T00:00:00Z'::timestamptz, '00000000-0000-0000-0000-000000000000'::uuid) +ORDER BY created_at, id +LIMIT 51; +""") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..b3c10c6 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,191 @@ +import base64 +import io +import uuid + +import httpx +from PIL import Image + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "att_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "A", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + cred = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def _idem(actor: dict) -> dict: + return {**_auth(actor), "Idempotency-Key": uuid.uuid4().hex} + + +def _png(w: int = 320, h: int = 200, color=(255, 0, 0)) -> bytes: + buf = io.BytesIO() + Image.new("RGB", (w, h), color).save(buf, format="PNG") + return buf.getvalue() + + +def _chat(client: httpx.Client, owner: dict, *members: dict) -> dict: + member_ids = [owner["id"]] + [m["id"] for m in members] + r = client.post( + f"{BASE}/api/v1/chats", + json={"title": "att", "member_ids": member_ids}, + headers=_auth(owner), + ) + assert r.status_code == 201, r.text + return r.json() + + +def _upload(client, actor, chat_id, data, filename="x.png", content_type="image/png"): + return client.post( + f"{BASE}/api/v1/chats/{chat_id}/attachments", + params={"filename": filename}, + content=data, + headers={**_auth(actor), "Content-Type": content_type}, + ) + + +def test_oversize_returns_413(): + with httpx.Client(timeout=30) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10 * 1024 * 1024) + r = _upload(c, a, chat["id"], big, filename="big.png") + assert r.status_code == 413, r.text + assert r.json()["error"]["code"] == "payload_too_large" + + +def test_corrupted_image_returns_415(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + data = _png()[:50] + r = _upload(c, a, chat["id"], data) + assert r.status_code == 415, r.text + assert r.json()["error"]["code"] == "unsupported_media_type" + + +def test_mime_spoof_returns_415(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + r = _upload(c, a, chat["id"], b"not an image at all") + assert r.status_code == 415, r.text + assert r.json()["error"]["code"] == "unsupported_media_type" + + +def test_bad_filename_returns_400(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + r = _upload(c, a, chat["id"], _png(), filename="../etc/passwd") + assert r.status_code == 400, r.text + assert r.json()["error"]["code"] == "invalid_request" + + +def test_empty_file_returns_415(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + r = _upload(c, a, chat["id"], b"") + assert r.status_code == 415, r.text + + +def test_outsider_cannot_download(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + eve = _register_and_login(c) + chat = _chat(c, a, b) + att = _upload(c, a, chat["id"], _png()).json() + r = c.get(f"{BASE}/api/v1/attachments/{att['id']}", headers=_auth(eve)) + assert r.status_code == 404 + r = c.get(f"{BASE}/api/v1/attachments/{att['id']}/content", headers=_auth(eve)) + assert r.status_code == 404 + + +def test_attach_foreign_attachment_returns_404(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat1 = _chat(c, a, b) + chat2 = _chat(c, a, b) + att = _upload(c, a, chat1["id"], _png()).json() + r = c.post( + f"{BASE}/api/v1/chats/{chat2['id']}/messages", + json={"text": "hi", "attachment_ids": [att["id"]]}, + headers=_idem(a), + ) + assert r.status_code == 404, r.text + + +def test_empty_text_without_attachments_returns_400(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": ""}, + headers=_idem(a), + ) + assert r.status_code == 400, r.text + assert r.json()["error"]["code"] == "invalid_request" + + +def test_empty_text_with_attachment_returns_201(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + att = _upload(c, a, chat["id"], _png()).json() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [att["id"]]}, + headers=_idem(a), + ) + assert r.status_code == 201, r.text + assert r.json()["attachment_ids"] == [att["id"]] + + +def test_user_error_is_not_500(): + with httpx.Client(timeout=10) as c: + a = _register_and_login(c) + b = _register_and_login(c) + chat = _chat(c, a, b) + cases = [ + (_upload(c, a, chat["id"], b"not image"), 415), + (_upload(c, a, chat["id"], b""), 415), + (_upload(c, a, chat["id"], _png(), filename="../x"), 400), + ( + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": ""}, + headers=_idem(a), + ), + 400, + ), + ] + for response, expected in cases: + assert response.status_code == expected, response.text + assert response.status_code < 500 \ No newline at end of file diff --git a/student-tests/test_cache.py b/student-tests/test_cache.py new file mode 100644 index 0000000..26b5bbf --- /dev/null +++ b/student-tests/test_cache.py @@ -0,0 +1,129 @@ +import base64 +import subprocess +import time +import uuid + +import httpx + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def _register_and_login(client: httpx.Client) -> dict: + username = "cch_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Cache Test", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def _auth_idem(actor: dict) -> dict: + return { + "Authorization": "Bearer " + actor["token"], + "Idempotency-Key": uuid.uuid4().hex, + } + + +def test_read_after_write_sees_new_message(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "raw", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + before = r.json()["items"] + assert len(before) == 0 + + text = "read-after-write-" + uuid.uuid4().hex[:8] + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=_auth_idem(alice), + ) + assert r.status_code == 201, r.text + + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + after = r.json()["items"] + texts = [m["text"] for m in after] + assert text in texts, f"read-after-write failed: {texts}" + + +def test_cache_recovers_after_valkey_restart(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "restart", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + for i in range(5): + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i}"}, + headers=_auth_idem(alice), + ) + assert r.status_code == 201, r.text + + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + before = r.json()["items"] + assert len(before) == 5 + + subprocess.run(COMPOSE + ["restart", "valkey"], check=True) + time.sleep(3) + + for _ in range(30): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + break + except Exception: + pass + time.sleep(1) + + with httpx.Client() as c: + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + after = r.json()["items"] + assert len(after) == 5 + assert [m["id"] for m in after] == [m["id"] for m in before] \ No newline at end of file diff --git a/student-tests/test_presigned.py b/student-tests/test_presigned.py new file mode 100644 index 0000000..ff04f94 --- /dev/null +++ b/student-tests/test_presigned.py @@ -0,0 +1,134 @@ +import base64 +import hashlib +import io +import uuid + +import httpx +from PIL import Image + +BASE = "http://localhost:8080" + + +def _user(c): + u = "ps_" + uuid.uuid4().hex[:10] + p = "Test-password-1234" + r = c.post(f"{BASE}/api/v1/users", json={"username": u, "display_name": "P", "password": p}) + assert r.status_code == 201, r.text + user = r.json() + cred = base64.b64encode(f"{u}:{p}".encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}).json() + return {"id": user["id"], "token": s["token"]} + + +def _auth(a): + return {"Authorization": "Bearer " + a["token"]} + + +def _png(w=320, h=200): + buf = io.BytesIO() + Image.new("RGB", (w, h), (0, 255, 0)).save(buf, format="PNG") + return buf.getvalue() + + +def _chat(c, a, b): + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "ps", "member_ids": [a["id"], b["id"]]}, + headers=_auth(a), + ) + assert r.status_code == 201, r.text + return r.json() + + +def test_presigned_happy_path(): + with httpx.Client(timeout=30) as c: + a = _user(c) + b = _user(c) + chat = _chat(c, a, b) + data = _png() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments/presign", + params={"filename": "p.png", "size": len(data), "content_type": "image/png"}, + headers=_auth(a), + ) + assert r.status_code == 201, r.text + presign = r.json() + put = httpx.put(presign["upload_url"], content=data, headers={"Content-Type": "image/png"}) + assert put.status_code == 200, put.text + r = c.post( + f"{BASE}/api/v1/attachments/{presign['attachment_id']}/complete", + headers=_auth(a), + ) + assert r.status_code == 201, r.text + att = r.json() + assert att["status"] == "ready" + assert att["sha256"] == hashlib.sha256(data).hexdigest() + dl = c.get(f"{BASE}/api/v1/attachments/{att['id']}/content", headers=_auth(b)) + assert dl.status_code == 200 + assert dl.content == data + + +def test_presigned_bad_signature_rejected(): + with httpx.Client(timeout=30) as c: + a = _user(c) + b = _user(c) + chat = _chat(c, a, b) + data = _png() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments/presign", + params={"filename": "p.png", "size": len(data), "content_type": "image/png"}, + headers=_auth(a), + ) + url = r.json()["upload_url"] + bad = url.replace("X-Amz-Signature=", "X-Amz-Signature=deadbeef") + put = httpx.put(bad, content=data, headers={"Content-Type": "image/png"}) + assert put.status_code == 403, put.text + + +def test_presigned_complete_without_upload_409(): + with httpx.Client(timeout=10) as c: + a = _user(c) + b = _user(c) + chat = _chat(c, a, b) + data = _png() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments/presign", + params={"filename": "p.png", "size": len(data), "content_type": "image/png"}, + headers=_auth(a), + ) + att_id = r.json()["attachment_id"] + r = c.post(f"{BASE}/api/v1/attachments/{att_id}/complete", headers=_auth(a)) + assert r.status_code == 409, r.text + + +def test_presigned_outsider_404(): + with httpx.Client(timeout=10) as c: + a = _user(c) + b = _user(c) + eve = _user(c) + chat = _chat(c, a, b) + data = _png() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments/presign", + params={"filename": "p.png", "size": len(data), "content_type": "image/png"}, + headers=_auth(a), + ) + att_id = r.json()["attachment_id"] + r = c.post(f"{BASE}/api/v1/attachments/{att_id}/complete", headers=_auth(eve)) + assert r.status_code == 404, r.text + + +def test_proxy_endpoint_still_works(): + with httpx.Client(timeout=10) as c: + a = _user(c) + b = _user(c) + chat = _chat(c, a, b) + data = _png() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/attachments", + params={"filename": "proxy.png"}, + content=data, + headers={**_auth(a), "Content-Type": "image/png"}, + ) + assert r.status_code == 201, r.text + assert r.json()["status"] == "ready" \ No newline at end of file diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 0000000..bd0a7bb --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,84 @@ +import base64 +import uuid +from concurrent.futures import ThreadPoolExecutor +import httpx + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Тест", + "password": password, + }, + ) + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + session = r.json() + return {"id": user["id"], "token": session["token"]} + + +def test_concurrent_writes_are_not_lost(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "race", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + def worker(worker_id: int): + with httpx.Client(timeout=10) as c: + for i in range(10): + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"w{worker_id}-m{i}"}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + if r.status_code != 201: + print(f"POST FAIL w{worker_id}-m{i}: {r.status_code} {r.text[:200]}") + + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(worker, range(10))) + + with httpx.Client() as c: + all_ids = [] + cursor = None + while True: + params = {"limit": 100} + if cursor: + params["cursor"] = cursor + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + params=params, + headers={"Authorization": "Bearer " + alice["token"]}, + ) + page = r.json() + if r.status_code != 200 or "items" not in page: + print(f"GET FAIL: {r.status_code} {r.text[:500]}") + print(f"chat_id: {chat['id']}") + print(f"alice_token: {alice['token'][:30]}...") + raise AssertionError(f"GET messages failed: {r.status_code}") + all_ids.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + + print(f"Got {len(all_ids)} messages, unique: {len(set(all_ids))}") + assert len(all_ids) == 100 + assert len(set(all_ids)) == 100 \ No newline at end of file diff --git a/student-tests/test_resilience.py b/student-tests/test_resilience.py new file mode 100644 index 0000000..4815fd9 --- /dev/null +++ b/student-tests/test_resilience.py @@ -0,0 +1,153 @@ +"""Automatic resilience checks: expiry, Redis/DB unavailability. + +Requires Docker: test stops the service, checks API behaviour, restarts it. +Run: pytest student-tests/test_resilience.py -v -s +""" +import base64 +import subprocess +import time +import uuid + +import httpx + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def _register_and_login(client: httpx.Client) -> dict: + username = "res_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Res", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return { + "id": user["id"], + "username": username, + "password": password, + "token": session["token"], + } + + +def _wait_ready(timeout: float = 60.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready in time") + + +def _compose(*args: str) -> None: + subprocess.run(COMPOSE + list(args), check=True) + + +def test_expiry_invalidates_session(monkeypatch): + """Session expires when SESSION_TTL_SECONDS is very small.""" + _compose("up", "-d", "--build", "api") + _wait_ready() + + with httpx.Client(timeout=5) as c: + actor = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + actor["token"]}, + ) + assert r.status_code == 200 + + # Restart API with 2-second TTL + _compose("stop", "api") + subprocess.run( + ["docker", "compose", "--env-file", ".env", "rm", "-f", "api"], + check=False, + ) + subprocess.run( + [ + "docker", "compose", "--env-file", ".env", + "run", "-d", "--name", "api-short-ttl", + "-e", "SESSION_TTL_SECONDS=2", + "-p", "127.0.0.1:8080:8080", + "api", + ], + check=True, + ) + _wait_ready() + + with httpx.Client(timeout=5) as c: + fresh = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + fresh["token"]}, + ) + assert r.status_code == 200 + print("Waiting 4 seconds for expiry...") + time.sleep(4) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + fresh["token"]}, + ) + assert r.status_code == 401, f"expected 401 after expiry, got {r.status_code}" + + subprocess.run(["docker", "stop", "api-short-ttl"], check=False) + subprocess.run(["docker", "rm", "api-short-ttl"], check=False) + _compose("up", "-d", "--build", "api") + _wait_ready() + + +def test_redis_unavailable_returns_503(): + """When Valkey is down, /health/ready returns 503.""" + _compose("up", "-d", "--build") + _wait_ready() + + _compose("stop", "valkey") + time.sleep(3) + + try: + with httpx.Client(timeout=5) as c: + r = c.get(f"{BASE}/health/ready") + assert r.status_code == 503, f"expected 503, got {r.status_code}" + body = r.json() + assert body["error"]["code"] == "unavailable" + finally: + _compose("start", "valkey") + _wait_ready() + + +def test_postgres_unavailable_returns_503(): + """When PostgreSQL is down, /health/ready returns 503.""" + _compose("up", "-d", "--build") + _wait_ready() + + _compose("stop", "postgres") + time.sleep(3) + + try: + with httpx.Client(timeout=5) as c: + r = c.get(f"{BASE}/health/ready") + assert r.status_code == 503, f"expected 503, got {r.status_code}" + body = r.json() + assert body["error"]["code"] == "unavailable" + finally: + _compose("start", "postgres") + _wait_ready() \ No newline at end of file diff --git a/student-tests/test_validation.py b/student-tests/test_validation.py new file mode 100644 index 0000000..299823d --- /dev/null +++ b/student-tests/test_validation.py @@ -0,0 +1,264 @@ +import base64 +import uuid +import httpx + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Тест", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return { + "id": user["id"], + "username": username, + "password": password, + "token": session["token"], + } + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def _auth_idem(actor: dict) -> dict: + return { + "Authorization": "Bearer " + actor["token"], + "Idempotency-Key": uuid.uuid4().hex, + } + + +def test_invalid_json_returns_400(): + with httpx.Client() as c: + r = c.post( + f"{BASE}/api/v1/users", + content=b"{not json at all", + headers={"Content-Type": "application/json"}, + ) + assert r.status_code == 400 + body = r.json() + assert body["error"]["code"] == "invalid_request" + assert "X-Request-Id" in r.headers + + +def test_duplicate_username_returns_409(): + with httpx.Client() as c: + actor = _register_and_login(c) + r = c.post( + f"{BASE}/api/v1/users", + json={ + "username": actor["username"], + "display_name": "Другой", + "password": "Another-password-123", + }, + ) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "conflict" + + +def test_empty_message_returns_400(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": ""}, + headers=_auth_idem(alice), + ) + assert r.status_code == 400 + + +def test_outsider_gets_404_not_403(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + eve = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}", + headers=_auth(eve), + ) + assert r.status_code == 404 + r2 = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "вторжение"}, + headers=_auth_idem(eve), + ) + assert r2.status_code == 404 + + +def test_pagination_walks_all_pages(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + sent_ids = [] + for i in range(5): + m = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i}"}, + headers=_auth_idem(alice), + ).json() + sent_ids.append(m["id"]) + + collected = [] + cursor = None + while True: + params = {"limit": 2} + if cursor: + params["cursor"] = cursor + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + params=params, + headers=_auth(alice), + ) + assert r.status_code == 200, r.text + page = r.json() + collected.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + assert collected == sent_ids + + +def test_wrong_password_returns_401(): + with httpx.Client() as c: + actor = _register_and_login(c) + bad = base64.b64encode( + (actor["username"] + ":wrong-password-1234").encode("utf-8") + ).decode("ascii") + r = c.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + bad}, + ) + assert r.status_code == 401 + assert "WWW-Authenticate" in r.headers + assert r.headers["WWW-Authenticate"].lower().startswith("basic") + + +def test_logout_invalidates_token(): + with httpx.Client() as c: + actor = _register_and_login(c) + r = c.delete( + f"{BASE}/api/v1/auth/sessions/current", + headers=_auth(actor), + ) + assert r.status_code == 204 + r = c.get(f"{BASE}/api/v1/users/me", headers=_auth(actor)) + assert r.status_code == 401 + + +def test_x_user_id_ignored_with_bearer(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={ + "Authorization": "Bearer " + alice["token"], + "X-User-Id": bob["id"], + }, + ) + assert r.status_code == 200 + assert r.json()["id"] == alice["id"] + + +def test_basic_on_business_endpoint_returns_401(): + with httpx.Client() as c: + actor = _register_and_login(c) + credential = base64.b64encode( + (actor["username"] + ":" + actor["password"]).encode("utf-8") + ).decode("ascii") + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 401 + + +def test_race_duplicate_username_only_one_wins(): + from concurrent.futures import ThreadPoolExecutor + + username = "race_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + + def register(): + with httpx.Client(timeout=10) as c: + return c.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Race", + "password": password, + }, + ).status_code + + with ThreadPoolExecutor(max_workers=2) as pool: + statuses = list(pool.map(lambda _: register(), range(2))) + + assert sorted(statuses) == [201, 409], f"got statuses {statuses}" + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + with httpx.Client() as c: + r = c.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201 + assert r.json()["user"]["username"] == username + + +def test_rollback_chat_with_unknown_member(): + with httpx.Client() as c: + alice = _register_and_login(c) + fake_id = str(uuid.uuid4()) + r = c.post( + f"{BASE}/api/v1/chats", + json={ + "title": "Не должно создаться", + "member_ids": [alice["id"], fake_id], + }, + headers=_auth(alice), + ) + assert r.status_code == 400, r.text + + chats = c.get( + f"{BASE}/api/v1/chats?limit=100", + headers=_auth(alice), + ).json() + titles = [ch["title"] for ch in chats["items"]] + assert "Не должно создаться" not in titles \ No newline at end of file