diff --git a/.env.example b/.env.example index d99d26a2..e06560a7 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 @@ -18,3 +18,12 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Public LB port on the host. If 8080 is busy, set e.g. 8081 and run make test with BASE_URL=http://localhost:8081 +LB_PORT=8080 +# Image tag of the api/migrate/s3-init services +APP_VERSION=dev +# Janitor of abandoned uploads (S3 PUT done, COMMIT never happened) +UPLOAD_GRACE_SECONDS=900 +JANITOR_INTERVAL_SECONDS=60 +# Experiments only, empty in normal operation: crash_after_s3_put | db_error_after_s3_put +FAULT_INJECTION= diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..213707e8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Скрипты и конфигурации исполняются в Linux-контейнерах: всегда LF. +*.sh text eol=lf +*.cfg text eol=lf +*.sql text eol=lf +*.py text eol=lf +*.yaml text eol=lf +Dockerfile* text eol=lf +Makefile text eol=lf diff --git a/.gitignore b/.gitignore index b5a71896..e216ea04 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +# Local secrets (scripts/init-secrets.sh) +infra/secrets/*.txt +ЗАЩИТА.md \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ef56e3f9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# Messenger API (lab4). One image for the "api", "migrate" and "s3-init" services. +FROM python:3.12.7-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY scripts/migrate.py scripts/s3_bootstrap.py ./scripts/ +COPY migrations ./migrations + +RUN useradd --create-home --uid 10001 appuser +USER appuser + +EXPOSE 8080 + +# Exec form: python is PID 1 and receives SIGTERM from "docker stop" itself. +# (lab2 used "sh -c uvicorn ...": sh stayed PID 1, uvicorn never got SIGTERM +# and was killed together with the container instead of draining.) +CMD ["python", "-m", "app.server"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..ef7a217a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,154 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Максимова Мария, БВТ2303 +- Целевая оценка; выбранный трек 5: 4 (трек не выбирается) +- Commit/PR: ветка `solution/maksimova_bvt2303` в fork https://gitea.privcloud.tech/Mariia_Maksimova/mtusi2026lab4, PR в `mtusi2026/mtusi2026lab4` (main) +- Использованные источники, библиотеки и помощь ИИ: FastAPI, SQLAlchemy/asyncpg, argon2-cffi, redis-py, aiobotocore/botocore, Pillow (только для JPEG); SeaweedFS, PostgreSQL, Valkey, HAProxy. Документация: AWS S3 PutObject/SigV4, SeaweedFS wiki (S3 API, `s3.json` identities), спецификация PNG (RFC 2083), HAProxy 3.0 configuration manual. Помощь ИИ (Claude Code): ревью и доработка кода вложений, написание тестов в `student-tests/`, проведение экспериментов и черновик этого отчёта; все результаты получены реальными прогонами на стенде и проверены. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверено на: Windows 11 + Docker Desktop 29.3.0 (WSL2, kernel 6.6.87, 14 vCPU, 7.4 GiB RAM), Docker Compose 5.1.0, Python 3.13 на хосте (для `make test`; достаточно 3.9+). + +| Роль т | Service т | Образ | Health / readiness | Volume | Сеть / порт | Credentials | +|---------------------|-----------|--------------------------------------------|-----------------------------------------------------------|-----------------------------------------|------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------| +| LB | `lb` | `haproxy:3.0.11-alpine` | `/lb-health`; backend-check `GET /health/ready` раз в 1 s | — | edge; `${LB_PORT:-8080}` на хосте, stats только `127.0.0.1:8404` | — | +| API × 2 | `api` | сборка `Dockerfile` (`python:3.12.7-slim`) | `/health/ready` (PostgreSQL + не в drain) | нет | edge + backend, порт не публикуется | Docker secrets: `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` | +| PostgreSQL | `db` | `postgres:16.4-alpine` | `pg_isready` | `postgres_data` | backend (internal) | `postgres_password` | +| Valkey (кэш сессий) | `valkey` | `valkey/valkey:8.0.1-alpine` | `PING` | нет (кэш, источник истины — PostgreSQL) | backend | `valkey_password` | +| S3 | `s3` | `chrislusf/seaweedfs:3.97` | `GET :9000/healthz` | `s3_data` | backend, порт не публикуется; master/filer/volume слушают только `127.0.0.1` внутри контейнера | admin + API ключи из secrets подставляются в `s3.json` при старте | +| Bootstrap bucket | `s3-init` | образ API | one-shot, `service_completed_successfully` | — | backend | единственный, кто получает admin-ключ S3 | +| Миграции | `migrate` | образ API | one-shot | — | backend | `database_url` | + +```sh +sh scripts/init-secrets.sh # создаёт infra/secrets/*.txt (случайные значения, не перезаписывает, не печатает) +cp .env.example .env # при занятом 8080 задайте LB_PORT=8081 +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# порядок: db, valkey, s3 -> healthy -> migrate (SQL migrations/*.sql, повтор = SKIP) +# s3-init (bucket + самопроверки приватности/прав, повтор = keep) -> api x2 healthy -> lb +curl -fsS http://localhost:${LB_PORT:-8080}/health/ready +make test BASE_URL=http://localhost:${LB_PORT:-8080} # на Windows: PYTHONUTF8=1 python tests/smoke.py --base-url ... + +# hook-тесты и собственные проверки — в Linux-контейнере tester (docker CLI + docker.sock): +docker compose --profile tools run --rm tester make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://lb:8080 +docker compose --profile tools run --rm tester make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://lb:8080 +docker compose --profile tools run --rm tester sh scripts/start-api.sh # вернуть остановленную реплику +docker compose --profile tools run --rm tester python3 student-tests/.py + +docker compose stop # безопасная остановка, данные сохраняются +docker compose down -v # НАМЕРЕННОЕ удаление всех учебных данных (volumes) +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client -> HAProxy (lb) -> api x2 --+--> PostgreSQL: users, chats, messages, sessions, idempotency_keys, + | attachments (метаданные), message_attachments, upload_intents + +--> Valkey: кэш сессий (tombstone при logout) + +--> SeaweedFS S3: приватный bucket messenger-attachments/originals/ +``` + +Перенесено из lab3 без изменения контракта: регистрация/Basic→Bearer, сессии в PostgreSQL с кэшем в Valkey, идемпотентная отправка сообщений, `X-Instance-Id`, graceful drain, HAProxy с active check, hooks `restart.sh`/`stop-one.sh`. Изменено: `CreateMessage` принимает `attachment_ids` (по умолчанию `[]`) и `text=""` при наличии вложений; fingerprint идемпотентности без вложений совпадает с lab3, так что старые ключи повторяются корректно. + +Модель. `attachments` (id, chat_id, owner_id, filename — только для отображения, content_type, size, sha256, width/height, status=`ready`, `object_key UNIQUE`) привязано к чату при загрузке. `message_attachments(message_id, attachment_id, position)` хранит порядок; одно вложение можно сослать из нескольких сообщений чата. Миграция `004_attachments.sql` — expand-only. + +Upload (`app/attachments.py`), всё до сохранения байтов: +1. filename (400: `/`, `\`, NUL, Cc/Cf/bidi, пустое/только точки, >255) → членство в чате (404) → `Content-Type` ровно `image/png|image/jpeg` (415, multipart не принимается) → `Content-Length > 10 MiB` → 413 **без чтения тела**; +2. слот загрузки: ≤ `MAX_CONCURRENT_UPLOADS=8` на реплику, ожидание ≤10 s, иначе 429; +3. тело читается потоково в `SpooledTemporaryFile` (в RAM ≤ 1 MiB на загрузку), считается размер и SHA-256 на лету; превышение 10 MiB в chunked → 413; приём дольше 60 s → 400; +4. проверка содержимого в потоке: формат по magic bytes и совпадение с заявленным MIME; PNG — собственный потоковый валидатор (CRC каждого чанка, IHDR, распаковка IDAT кусками по 64 KiB с подсчётом и сравнением с ожидаемым размером, без хранения пикселей); лимит 40 млн пикселей проверяется по IHDR/SOF **до** распаковки (413); JPEG — Pillow `draft()` 1/8 + `load()` (повреждённый поток → 415); +5. `INSERT upload_intent` + COMMIT → S3 PUT (SigV4 подписывает SHA-256 тела) → одна транзакция «DELETE intent + INSERT attachment» → **201 ready**. Честный 201 возможен только после этого COMMIT: и байты, и метаданные уже устойчивы. + +Download: авторизация по `chat_members` в PostgreSQL (чужое/неизвестное → 404), затем потоковая отдача из S3 через API, без redirect, с истинным `Content-Type`, `Content-Length`, `nosniff`, `Cache-Control: private, no-store`. Приватный bucket не заменяет ACL: случайный ключ можно утечь через лог, а права участника чата знает только приложение. + +Согласованность БД/S3 (ответ на «PUT прошёл, а COMMIT нет»): + +| Отказ | Что делает система | +|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| PUT упал / истёк дедлайн | 503 сразу. Исход PUT неизвестен (S3 может дописать объект **после** нашего DELETE), поэтому объект удаляется best effort в фоне, а intent **остаётся**; janitor удалит объект ещё раз после grace и только потом intent | +| COMMIT упал | 503 сразу. Исход COMMIT тоже неизвестен (мог пройти, а потерялся ответ). Фоновая компенсация берёт `SELECT … FOR UPDATE` intent: intent нет → COMMIT прошёл, объект не трогаем; иначе DELETE объекта → DELETE intent | +| Процесс умер после PUT | intent остался; janitor каждой реплики (`JANITOR_INTERVAL_SECONDS=60`, jitter) через `UPLOAD_GRACE_SECONDS=900` удаляет объект и intent; `FOR UPDATE SKIP LOCKED` — реплики не мешают друг другу и финализирующейся загрузке | +| Строка без объекта (потеря байтов вне приложения) | download → 500 `internal_error` и ERROR в логе; метаданные не удаляются (сообщения на них ссылаются) | + +Инвариант: объект без строки существует, только пока жив его intent; объект закоммиченной строки никогда не удаляется. Grace 900 s ≫ максимального времени upload (60 s приём + 16 s PUT). + +Недоступность S3. `/health/ready` S3 не проверяет: без S3 деградируют только вложения (503), чаты и сообщения работают; иначе отказ S3 вывел бы из LB все реплики. Каждый S3-вызов ограничен общим дедлайном `S3_OPERATION_TIMEOUT_SECONDS=6` (DNS + повторы) — меньше `timeout server 10s` у LB; для PUT дедлайн `6 s + size / 1 MiB/s` (≤16 s для 10 MiB), а HAProxy даёт POST вложения 40 s. L7-повторы HAProxy для `/api/v1/attachments/` отключены: 503 там означает отказ общего S3, повтор на другой реплике бесполезен. + +Права S3 (`infra/seaweedfs/s3.json.template`): identity `admin` есть только у `s3-init`; identity `messenger-api` — `Read/Write/List` только на `messenger-attachments` (Write нужен для DELETE janitor-а). Анонимного доступа нет. Admin UI/API SeaweedFS (master 9333, filer 8888, volume 8080, gRPC) слушают только `127.0.0.1` внутри контейнера. + +*Соответствие `.env.example`:** `APP_HOST/APP_PORT`, `SESSION_TTL_SECONDS`, `DATABASE_URL_FILE`, `SESSION_STORE=valkey`, `CACHE_URL_FILE`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS`, `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`, `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` читаются `app/config.py` под теми же именами. Дополнительно в `compose.yaml` (`x-api-env`): `S3_OBJECT_PREFIX`, `S3_CONNECT/READ/OPERATION_TIMEOUT_SECONDS`, `S3_PUT_MIN_BYTES_PER_SECOND`, `UPLOAD_SPOOL_MEMORY_BYTES`, `UPLOAD_TIMEOUT_SECONDS`, `MAX_CONCURRENT_UPLOADS`, `UPLOAD_SLOT_WAIT_SECONDS`, `UPLOAD_GRACE_SECONDS`, `JANITOR_INTERVAL_SECONDS`, `FAULT_INJECTION` (только для экспериментов, пусто в норме). `APP_ENV` не используется. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Все файлы — в `evidence/`, тесты — в `student-tests/`; прогоны 2026-10-30 на стенде выше. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +|-----------------------------------------------------------------------------------------------|-------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------| +| Публичный smoke | `make test BASE_URL=http://localhost:8081` | `PASS: lab4 public API scenario` (SHA-256, 404 постороннему) | `smoke.txt` | +| 3: S3 в стенде, метаданные в PG, байты в приватном bucket | `student-tests/s3_failures.py`; `api-no-local-files.txt` | object_key только в PG; анонимные GET/HEAD/LIST/PUT/DELETE напрямую в S3 → 403; у API нет volumes, `/tmp` пуст | `s3-failures.txt`, `api-no-local-files.txt` | +| 3: PNG/JPEG ≤10 MiB, ≤40 Мпикс, MIME по содержимому, опасные filename | `student-tests/attachment_edge_cases.py` | 60/60 OK: ровно 10 MiB и ровно 40 Мпикс → 201; +1 байт → 413; бомба 50 Мпикс (~10 KB) → 413 | `attachment-edge-cases.txt` | +| 3: upload → стоп обслужившей реплики → download через другую | `make test-failover` ×2; `student-tests/upload_failover.py` | failover: 0 ошибок, восстановление записи 4.0/4.2 s; для **каждой** из двух реплик SHA-256 совпал после её остановки | `failover.txt`, `upload-failover.txt` | +| 3: restart с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; те же ID контейнеров db/s3/valkey/api/lb с новым `StartedAt`, volumes на месте | `persistence.txt` | +| 4: тесты большого файла, повреждения, подмены MIME, чужого id, вложения другого чата; нет 500 | `attachment_edge_cases.py` | обрезка/CRC/мусор в zlib при верном CRC/без IEND → 415; PNG как JPEG, JPEG как PNG, text/plain, multipart → 415; вложение чужого и своего другого чата → 404; ни одного 500 | `attachment-edge-cases.txt` | +| 4: компенсация/сверка/очистка с grace | `student-tests/upload_compensation.py` | DB error после PUT → 503 + фоновая компенсация, intent не остался; crash после PUT → intent + объект, janitor до grace их не трогает, после grace удаляет оба; свежий intent и готовое вложение целы | `upload-compensation.txt` | +| 4: лимиты RAM, ≥10 одновременных загрузок | `student-tests/upload_memory.py --concurrency 12/20` | 12 × 9.5 MiB: прирост пика RSS 13–14 MiB на реплику (наивно 57 MiB данных на реплику) | `upload-memory-c12.json`, `upload-memory-c20.json` | +| 4: идемпотентный bootstrap, отдельные ограниченные credentials | `docker compose run --rm s3-init` ×2, `migrate` | `keep bucket`, все самопроверки PASS: API-ключ не создаёт/не удаляет bucket, не видит чужих; миграции SKIP | `bootstrap-idempotent.txt` | +| 4: админ-консоль закрыта, private bucket проверен вне API | `s3_failures.py` из сети стенда | порты 9333/8888/8080/19333/18888/23646 недоступны; S3 не опубликован на хосте; анонимные запросы → 403 | `s3-failures.txt` | +| 3/4: недоступность S3 | `s3_failures.py` (docker stop s3) | ready 200, сообщения 201, metadata 200; upload/download → 503 за 6.0–6.2 s; 415 остаётся 415; после старта S3 всё работает, старый файл цел | `s3-failures.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Память при одновременных загрузках (уровень 4) -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Гипотеза: пик памяти API определяется конструкцией (spool ≤1 MiB/загрузку, потоковая проверка PNG, ≤8 слотов), а не числом × размером файлов. +Методика: `upload_memory.py` перезапускает обе реплики (сброс состояния), N потоков стартуют одновременно (Barrier) и загружают валидный PNG из шума 9 962 537 байт (9.5 MiB, несжимаемый — худший случай для проверки IDAT); внутри каждой реплики и S3 каждые 100 мс читаются `VmRSS` процесса и `anon` cgroup. `VmHWM` не подходит: пик импорта библиотек при старте (~170 MiB) выше пика загрузок. Лимиты: API 0.5 CPU / 256 MiB, S3 1 CPU / 1 GiB. + +| N (на реплику) | Статусы | Wall, s | p50 / max, s | Пик RSS API − базовый (≈105 MiB) | Наивная оценка «всё в RAM» на реплику | Пик RSS SeaweedFS | +|------------------------|-----------|-----------|----------------|------------------------------------|-----------------------------------------|---------------------| +| 12 (6+6) | 12 × 201 | 5.2 | 5.0 / 5.2 | +12.9 / +13.7 MiB | 57 MiB | 546 MiB | +| 20 (10+10, > 8 слотов) | 20 × 201 | 9.9 | 8.6 / 9.9 | +16.5 / +17.7 MiB | 95 MiB | 779 MiB | + +Гипотеза для API подтверждена: прирост ~2 MiB на одновременную загрузку и почти не растёт сверх 8 слотов (лишние ждут слот; в одном из повторов при N=20 двое получили 429 после 10 s ожидания — ожидаемый backpressure). SHA-256 проверен. + +Побочная находка: узким местом оказался S3-сервер. С исходным лимитом 512 MiB SeaweedFS при 12 параллельных PUT дошёл до 519 MiB и был убит (контейнер перезапущен, все загрузки — 503). Исправлено: лимит 1 GiB и `GOMEMLIMIT=600MiB` (мягкий лимит Go GC); после нагрузки RSS возвращается к ~210 MiB. Это одно измерение на одном ноутбуке, а не ёмкость SeaweedFS вообще. + +### Компенсация отказа между S3 и БД (уровень 4) + +Найденный дефект (до исправления). Сверка содержимого bucket со строками БД на «грязном» стенде нашла 12 объектов без строки и без intent — то, что запрещено инвариантом. Причина: при истёкшем дедлайне PUT компенсация удаляла объект **и intent**, а SeaweedFS дописывал PUT после DELETE. Второй риск того же рода: при неоднозначном COMMIT немедленное удаление объекта оставило бы строку без байтов. Исправлено (см. таблицу отказов выше); 12 лишних объектов удалены. `upload_compensation.py` после исправления проверяет компенсацию при ошибке БД и очистку janitor-ом после падения процесса. + +### Отказы + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +|-----------------------------------------------------------------|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| +| `docker stop` обслужившей API-реплики (×2 + для каждой реплики) | запись и download через другую | 0 ошибок записи; файл скачан по SHA-256 через оставшуюся | первая успешная запись через 4.0–4.2 s; потерь нет | +| Restart lb + api + db + valkey + s3 с volumes | данные, сессия и оригинал сохранены | PASS с прежним токеном | hook и ожидание готовности уложились в лимиты теста (90 + 90 s), длительность отдельно не замерялась; потерь нет | +| `docker stop s3` | только вложения → 503, остальное работает | ready 200, message 201, metadata 200, upload/download 503 за ~6 s | сразу после старта S3; потерь нет, intents неудачных PUT убирает janitor | +| Падение процесса после PUT до COMMIT | клиент без 201, orphan не вечный | intent + объект → удалены janitor после grace | grace (900 s; 20 s в тесте) | +| Потеря БД после PUT | 503, без мусора | 503; фоновая компенсация удалила объект и intent | ~0.1 s | +| 12 параллельных PUT по 10 MiB при лимите S3 512 MiB | — | SeaweedFS убит по памяти, все 503 | исправлено лимитом 1 GiB + `GOMEMLIMIT` | +| Оборванная клиентом загрузка | реплика продолжает работу | следующая загрузка 201; по коду PUT начинается только после приёма всего тела, поэтому объекта нет (отдельно не измерялось) | — | + +## Ответы на вопросы + +Почему файл в Docker volume одной API-реплики не равен объектному хранилищу? +Volume принадлежит одному контейнеру на одном host. Файл, записанный репликой A, не видит реплика B: если A остановить (как в `make test-failover`), download через B вернёт 404 или 500, хотя метаданные в PostgreSQL есть. Чтобы реплики оставались взаимозаменяемыми, байты должны лежать в общем сетевом хранилище, доступном всем репликам. S3 — это отдельный сервис со своим API (PUT/GET/DELETE по ключу), правами доступа (identity `messenger-api` с доступом только к своему bucket), собственным жизненным циклом и масштабированием, его можно менять независимо от API. Поэтому у API в стенде вообще нет volumes, а `/tmp` пуст (`api-no-local-files.txt`); тело загрузки лежит только во временном spool на время запроса. + +Когда можно честно вернуть 201? +Только когда одновременно устойчивы и байты, и метаданные: S3 PUT подтверждён (SigV4 подписывает SHA-256 тела, значит S3 сохранил ровно эти байты), и закоммичена транзакция «DELETE intent + INSERT attachment». Если вернуть 201 раньше, например после PUT, но до COMMIT, клиент получит id, которого нет в БД; если до PUT — строка будет указывать на несуществующий объект. При любом сбое до этого COMMIT клиент получает 503, а не 201, и может безопасно повторить загрузку. + +Что произойдёт, если S3 PUT прошёл, а COMMIT в PostgreSQL — нет? +Объект в S3 есть, а строки `attachments` нет. Это orphan: он занимает место, но не является ошибкой данных, потому что перед PUT закоммичен `upload_intent` с его `object_key`. Клиент сразу получает 503. Дальше: +1. Фоновая компенсация берёт intent через `SELECT … FOR UPDATE`. Исход COMMIT неизвестен (он мог пройти, а потеряться только ответ), поэтому: intent нет → COMMIT на самом деле прошёл, объект не трогаем; intent есть → DELETE объекта, затем DELETE intent. +2. Если процесс умер и компенсация не успела, intent остаётся, и janitor любой реплики после `UPLOAD_GRACE_SECONDS=900` удаляет сначала объект, потом intent (`FOR UPDATE SKIP LOCKED`). +Порядок «сначала объект, потом intent» важен: если удалить intent первым и упасть, останется объект, о котором никто не знает. Инвариант: объект без строки существует только пока жив его intent, а объект закоммиченной строки никогда не удаляется. Проверено `upload_compensation.py`. + +Почему случайный object key и приватный bucket не заменяют ACL приложения? +Приватный bucket защищает только от анонимного доступа к S3. Он ничего не знает о пользователях и чатах: API ходит в S3 под одним сервисным ключом, который может читать любые объекты. Кто участник чата, знает только PostgreSQL (`chat_members`), поэтому проверять право на скачивание должно приложение при каждом запросе. Случайный ключ — это не право доступа, а секрет, который может утечь через лог, URL, ответ API, бэкап или бывшего участника чата, и отозвать его нельзя. Кроме того, права меняются со временем: пользователя удалили из чата, а ключ у него остался. Поэтому download идёт через API: сначала проверка членства в чате (чужое/неизвестное вложение → 404), потом потоковая отдача из S3, без redirect и presigned URL. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker host, один PostgreSQL, один SeaweedFS без репликации, один LB: выдерживается отказ одной API-реплики и перезапуск хранилищ с volumes, но не потеря host/диска/S3-тома. Резервного копирования нет. +- Нет отдельного reconciler: объект без строки и без intent (ручное вмешательство, баг) автоматически не удаляется; потеря байтов готового вложения видна только по 500 и ERROR в логе при download. +- Прогрессивные JPEG > 16 Мпикс проверяются только по структуре маркеров (полное декодирование держало бы все DCT-коэффициенты в памяти); такие повреждения внутри энтропийного потока не ловятся. +- `MAX_CONCURRENT_UPLOADS` — на реплику, не глобально; общий предел S3 задаётся числом реплик × 8. Ёмкость SeaweedFS при большем числе реплик не измерялась. +- Измерения памяти — по одному-два прогона на ноутбуке; при N=12 в разных прогонах прирост был от 3.6 до 13.7 MiB на реплику, абсолютные числа зависят от аллокатора и версии Python. \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 00000000..e45cd2c4 --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,441 @@ +"""Attachments: synchronous upload to S3, metadata in PostgreSQL (lab4). + +Upload, in this order (all checks before the bytes are stored): + + 1. filename (400) -> chat membership (404) -> Content-Type header (415) + -> Content-Length (413 without reading the body); + 2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429; + 3. the body is read in chunks into a SpooledTemporaryFile (only + UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more + than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400; + 4. content validation in a thread (app.images): format by magic bytes, + pixel limit from the header (413), integrity (415); + 5. upload intent COMMIT -> S3 PUT -> one transaction "delete intent + + insert ready attachment" -> 201. + +Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs +in the background): + - PUT failed or timed out: the outcome is unknown - S3 may still finish + writing the object AFTER our DELETE. So the object is deleted best + effort, but the intent is KEPT; the janitor deletes the object again + after UPLOAD_GRACE_SECONDS and only then the intent; + - COMMIT failed: the outcome is unknown too - COMMIT may have succeeded + and only the answer was lost. Under the intent's row lock: no intent -> + the row is committed, keep the object; else delete object, then intent; + - the process died after PUT: the intent stays; the janitor deletes the + object and the intent after UPLOAD_GRACE_SECONDS. +So an object without a row exists only while its intent exists, and an +object of a committed row is never deleted. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import random +import tempfile +import unicodedata +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote +from uuid import UUID, uuid4 + +from fastapi import Request +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.requests import ClientDisconnect + +from app import repositories +from app.config import settings +from app.db import SessionLocal +from app.errors import ( + ApiError, + InvalidRequest, + NotFound, + PayloadTooLarge, + RateLimited, + Unavailable, + UnsupportedMediaType, +) +from app.images import InvalidImage, TooManyPixels, validate_image +from app.storage import ObjectMissing, StorageUnavailable, storage + + +logger = logging.getLogger("messenger.attachments") + +ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"} +# Characters that can turn a display name into a path or hide its real +# extension: separators, NUL/control characters, bidi overrides etc. +FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"} + +upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads) +# Strong references to background compensation tasks (asyncio keeps only +# weak ones); if the process dies first, the intent stays for the janitor. +_compensations: set[asyncio.Task] = set() + + +def _to_attachment(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def validate_filename(filename: str) -> str: + if not 1 <= len(filename) <= 255 or not filename.strip(): + raise InvalidRequest("filename must be 1-255 characters, not blank.") + + if "/" in filename or "\\" in filename: + raise InvalidRequest("filename must not contain path separators.") + + if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename): + raise InvalidRequest("filename must not contain control characters.") + + if filename.strip(". ") == "": + raise InvalidRequest("filename must not be a relative path.") + + return filename + + +def declared_content_type(request: Request) -> str: + raw = request.headers.get("Content-Type", "") + media_type = raw.split(";", 1)[0].strip().lower() + + if media_type not in ALLOWED_CONTENT_TYPES: + raise UnsupportedMediaType( + "Content-Type must be image/png or image/jpeg (raw bytes, not multipart)." + ) + + return media_type + + +def _too_large(message: str) -> PayloadTooLarge: + error = PayloadTooLarge(message) + # The rest of the body is not read: the connection cannot be reused. + error.headers["Connection"] = "close" + return error + + +def check_declared_length(request: Request) -> None: + raw = request.headers.get("Content-Length") + if raw is None: + return # chunked: the limit is enforced while reading + + if not raw.isdigit(): + raise InvalidRequest("Invalid Content-Length.") + + if int(raw) > settings.max_upload_bytes: + raise _too_large("File is larger than 10 MiB.") + + +async def _receive_body(request: Request, spool) -> tuple[int, str]: + """Stream the body into the spool; return (size, sha256).""" + digest = hashlib.sha256() + size = 0 + + try: + async with asyncio.timeout(settings.upload_timeout_seconds): + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > settings.max_upload_bytes: + raise _too_large("File is larger than 10 MiB.") + digest.update(chunk) + spool.write(chunk) + except TimeoutError: + raise ApiError( + 400, + "invalid_request", + "Upload was not completed in time.", + {"Connection": "close"}, + ) from None + except ClientDisconnect: + # Nobody will read this answer; nothing was stored anywhere. + logger.info("upload interrupted by the client after %d bytes", size) + raise InvalidRequest("Upload was interrupted.") from None + + return size, digest.hexdigest() + + +async def _acquire_slot() -> None: + try: + async with asyncio.timeout(settings.upload_slot_wait_seconds): + await upload_slots.acquire() + except TimeoutError: + raise RateLimited("Too many concurrent uploads, retry later.", 2) from None + + +async def _discard_after_failed_put(object_key: str) -> None: + """Best effort only; the intent stays for the janitor (late PUT).""" + try: + await storage.delete(object_key) + except StorageUnavailable: + logger.warning("compensation: S3 delete failed, janitor will retry") + + +async def _compensate_failed_commit(object_key: str) -> None: + """Delete the object only if its upload really did not commit.""" + try: + async with SessionLocal() as db: + row = await repositories.lock_upload_intent(db, object_key) + if row is None: + # COMMIT succeeded after all (or the janitor finished it). + await db.rollback() + return + if not row["committed"]: + try: + await storage.delete(object_key) + except StorageUnavailable: + await db.rollback() + logger.warning("compensation: S3 delete failed, janitor will retry") + return + await repositories.delete_upload_intent(db, object_key) + except Exception as exc: + logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__) + + +async def upload_attachment( + request: Request, + db: AsyncSession, + *, + chat_id: UUID, + filename: str, + current_user: dict[str, Any], +) -> dict[str, Any]: + validate_filename(filename) + + chat = await repositories.get_accessible_chat( + db, chat_id=chat_id, user_id=current_user["id"] + ) + if chat is None: + raise NotFound() + # Do not keep a pooled connection idle in a transaction while the + # client uploads up to 10 MiB. + await db.rollback() + + content_type = declared_content_type(request) + check_declared_length(request) + + await _acquire_slot() + try: + with tempfile.SpooledTemporaryFile( + max_size=settings.upload_spool_memory_bytes + ) as spool: + size, sha256 = await _receive_body(request, spool) + if size == 0: + raise InvalidRequest("Request body with image bytes is required.") + + try: + info = await asyncio.to_thread( + validate_image, spool, content_type, settings.max_image_pixels + ) + except TooManyPixels: + raise PayloadTooLarge("Image has more than 40 million pixels.") from None + except InvalidImage as exc: + raise UnsupportedMediaType(str(exc)) from None + + return await _store( + db, + spool=spool, + chat_id=chat_id, + owner_id=current_user["id"], + filename=filename, + content_type=info.content_type, + size=size, + sha256=sha256, + width=info.width, + height=info.height, + ) + finally: + upload_slots.release() + + +async def _store( + db: AsyncSession, + *, + spool, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size: int, + sha256: str, + width: int, + height: int, +) -> dict[str, Any]: + attachment_id = uuid4() + # Server-generated random key, unrelated to the filename and to the + # public attachment ID. + object_key = f"{settings.s3_object_prefix}{uuid4().hex}" + + await repositories.create_upload_intent( + db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id + ) + + try: + await storage.put(object_key, spool, size, content_type) + except StorageUnavailable as exc: + logger.warning("S3 PUT failed: %s", exc) + _in_background(_discard_after_failed_put(object_key)) + raise Unavailable() from None + + if settings.fault_injection == "crash_after_s3_put": + logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT") + os._exit(1) + + try: + if settings.fault_injection == "db_error_after_s3_put": + raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT") + + row = await repositories.finalize_attachment( + db, + attachment_id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + content_type=content_type, + size_bytes=size, + sha256=sha256, + width=width, + height=height, + object_key=object_key, + ) + except Exception as exc: + logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__) + _in_background(_compensate_failed_commit(object_key)) + raise Unavailable() from None + + if row is None: + # The janitor already claimed this intent (upload slower than the + # grace period): the object is (being) deleted, report a failure. + logger.warning("upload intent expired before COMMIT") + raise Unavailable() + + return _to_attachment(row) + + +async def get_attachment( + db: AsyncSession, + *, + attachment_id: UUID, + current_user: dict[str, Any], +) -> dict[str, Any]: + row = await repositories.get_accessible_attachment( + db, attachment_id=attachment_id, user_id=current_user["id"] + ) + if row is None: + raise NotFound() + return _to_attachment(row) + + +async def open_attachment_content( + db: AsyncSession, + *, + attachment_id: UUID, + variant: str, + current_user: dict[str, Any], +) -> tuple[AsyncIterator[bytes], dict[str, str]]: + """Authorize in PostgreSQL, then stream the bytes from S3 (no redirect).""" + row = await repositories.get_accessible_attachment( + db, attachment_id=attachment_id, user_id=current_user["id"] + ) + if row is None: + raise NotFound() + await db.rollback() + + if variant != "original": + # Lab4 has no variants; a ready attachment without it -> 404. + raise NotFound() + + try: + chunks, length = await storage.open_download(row["object_key"]) + except StorageUnavailable as exc: + logger.warning("S3 GET failed: %s", exc) + raise Unavailable() from None + except ObjectMissing: + # Metadata says "ready" but the bytes are gone: an inconsistency, + # not a user error (logged as ERROR). + logger.error("object of attachment %s is missing in S3", attachment_id) + raise + + if length != row["size_bytes"]: + logger.error("object of attachment %s has a wrong size", attachment_id) + raise ObjectMissing(row["object_key"]) + + headers = { + "Content-Type": row["content_type"], + "Content-Length": str(length), + "Content-Disposition": "inline; filename*=UTF-8''" + + quote(row["filename"], safe=""), + "ETag": f'"{row["sha256"]}"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + } + return chunks, headers + + +def _in_background(compensation) -> None: + """Do not keep the client waiting for compensation: with S3 down each + attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503 + anyway.""" + task = asyncio.create_task(compensation) + _compensations.add(task) + task.add_done_callback(_compensations.discard) + + +# --- Janitor ---------------------------------------------------------------- + +async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int: + """Delete objects of upload intents older than the grace period. + + One intent per transaction: lock it (SKIP LOCKED), delete the object in + S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and + stop: the intent stays and the next run retries. + """ + cleaned = 0 + for _ in range(limit): + async with SessionLocal() as db: + row = await repositories.lock_stale_intent(db, grace_seconds) + if row is None: + await db.rollback() + break + + if not row["committed"]: + try: + await storage.delete(row["object_key"]) + except StorageUnavailable: + await db.rollback() + logger.warning("janitor: S3 unavailable, will retry") + break + + await repositories.delete_upload_intent(db, row["object_key"]) + cleaned += 1 + logger.warning( + "janitor: removed abandoned upload %s (object %s)", + row["object_key"], + "kept, row committed" if row["committed"] else "deleted", + ) + return cleaned + + +async def janitor_loop() -> None: + while True: + # Jitter: replicas do not all wake up at the same moment. + await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2)) + try: + await cleanup_stale_uploads(settings.upload_grace_seconds) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("janitor run failed: %s", type(exc).__name__) diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..d51862a7 --- /dev/null +++ b/app/cache.py @@ -0,0 +1,179 @@ +"""Valkey session cache. + +PostgreSQL is the source of truth for sessions. Valkey only keeps a short +lived copy of "token hash -> user" to avoid a SQL lookup on every request: + +- key: messenger:session:, the raw token is never stored; +- value TTL: min(SESSION_CACHE_TTL_SECONDS, remaining session lifetime), + and expires_at is checked again on every read; +- logout writes a "revoked" tombstone (SET ... EX, overwrite), while cache + population uses SET ... NX, so a concurrent request cannot resurrect a + revoked session in the cache; +- any Valkey error or timeout is a cache miss: requests go to PostgreSQL and + the cache is skipped for CACHE_RETRY_AFTER_SECONDS. After such an error, + and on every API process start, the cache is flushed before its next use, + because an invalidation may have been lost while Valkey was unreachable + (also by a previous API process). +""" +from __future__ import annotations + +import asyncio +import json +import logging +import math +import time +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +from redis.asyncio import ConnectionPool, Redis +from redis.exceptions import RedisError + +from app.config import settings + + +logger = logging.getLogger("messenger.cache") + +KEY_PREFIX = "messenger:session:" +REVOKED = "revoked" + + +class Revoked: + """Marker returned when the cache knows the session was revoked.""" + + +REVOKED_SESSION = Revoked() + + +def _key(token_hash: str) -> str: + return KEY_PREFIX + token_hash + + +def _serialize(user: dict[str, Any], expires_at: datetime) -> str: + return json.dumps( + { + "id": str(user["id"]), + "username": user["username"], + "display_name": user["display_name"], + "created_at": user["created_at"].isoformat(), + "expires_at": expires_at.isoformat(), + }, + separators=(",", ":"), + ) + + +def _deserialize(raw: str) -> tuple[dict[str, Any], datetime]: + payload = json.loads(raw) + user = { + "id": UUID(payload["id"]), + "username": payload["username"], + "display_name": payload["display_name"], + "created_at": datetime.fromisoformat(payload["created_at"]), + } + return user, datetime.fromisoformat(payload["expires_at"]) + + +class SessionCache: + def __init__(self, url: str | None) -> None: + self._client: Redis | None = None + self._unavailable_until = 0.0 + # A process never trusts a cache it has not flushed itself: an + # earlier process may have lost an invalidation (logout while + # Valkey was unreachable) and that knowledge died with it. + # Cleared only after a successful FLUSHDB. + self._needs_flush = True + + if url: + pool = ConnectionPool.from_url( + url, + max_connections=settings.cache_max_connections, + socket_timeout=settings.cache_timeout_seconds, + socket_connect_timeout=settings.cache_timeout_seconds, + decode_responses=True, + ) + self._client = Redis(connection_pool=pool) + + @property + def enabled(self) -> bool: + return self._client is not None + + async def _run(self, operation): + """Run one cache operation; return (ok, result).""" + if self._client is None: + return False, None + + if time.monotonic() < self._unavailable_until: + return False, None + + try: + async with asyncio.timeout(settings.cache_timeout_seconds * 2): + if self._needs_flush: + await self._client.flushdb() + self._needs_flush = False + logger.warning("session cache flushed before use") + return True, await operation(self._client) + except (RedisError, OSError, TimeoutError) as exc: + self._unavailable_until = ( + time.monotonic() + settings.cache_retry_after_seconds + ) + self._needs_flush = True + logger.warning( + "session cache unavailable, falling back to PostgreSQL: %s", + type(exc).__name__, + ) + return False, None + + async def get( + self, + token_hash: str, + ) -> dict[str, Any] | Revoked | None: + ok, raw = await self._run(lambda client: client.get(_key(token_hash))) + + if not ok or raw is None: + return None + + if raw == REVOKED: + return REVOKED_SESSION + + try: + user, expires_at = _deserialize(raw) + except (ValueError, KeyError, TypeError): + return None + + if expires_at <= datetime.now(timezone.utc): + return None + + return user + + async def put( + self, + token_hash: str, + user: dict[str, Any], + expires_at: datetime, + ) -> None: + remaining = (expires_at - datetime.now(timezone.utc)).total_seconds() + ttl = min(settings.session_cache_ttl_seconds, math.floor(remaining)) + + if ttl < 1: + return + + value = _serialize(user, expires_at) + await self._run( + lambda client: client.set(_key(token_hash), value, ex=ttl, nx=True) + ) + + async def revoke(self, token_hash: str) -> None: + await self._run( + lambda client: client.set( + _key(token_hash), + REVOKED, + ex=settings.session_cache_ttl_seconds, + ) + ) + + async def close(self) -> None: + if self._client is not None: + await self._client.aclose() + + +session_cache = SessionCache(settings.cache_url) diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..942941ea --- /dev/null +++ b/app/config.py @@ -0,0 +1,180 @@ +from dataclasses import dataclass +from pathlib import Path +import os +import socket + + +def read_value(name: str, default: str | None = None) -> str: + file_name = os.getenv(f"{name}_FILE") + + if file_name: + return Path(file_name).read_text(encoding="utf-8").strip() + + value = os.getenv(name, default) + + if value is None: + raise RuntimeError(f"{name} is required.") + + return value + + +def read_positive_int(name: str, default: str) -> int: + value = int(os.getenv(name, default)) + + if value < 1: + raise RuntimeError(f"{name} must be positive.") + + return value + + +def read_positive_float(name: str, default: str) -> float: + value = float(os.getenv(name, default)) + + if value <= 0: + raise RuntimeError(f"{name} must be positive.") + + return value + + +@dataclass(frozen=True) +class Settings: + app_host: str + app_port: int + database_url: str + session_ttl_seconds: int + # PostgreSQL connection pool and timeouts. + db_pool_size: int + db_max_overflow: int + db_pool_timeout_seconds: float + db_connect_timeout_seconds: float + db_statement_timeout_ms: int + # Valkey is a session cache; PostgreSQL stays the source of truth. + cache_url: str | None + session_cache_ttl_seconds: int + cache_timeout_seconds: float + cache_max_connections: int + cache_retry_after_seconds: float + # Lab3: X-Instance-Id is the container hostname (short Docker ID), stable + # for the process lifetime; scripts/stop-one.sh maps it to a container. + instance_id: str + idempotency_ttl_seconds: int + # Graceful shutdown: readiness answers 503 for shutdown_drain_seconds so + # the LB drops this replica, then uvicorn closes the listener and waits + # for in-flight requests up to shutdown_timeout_seconds. + shutdown_drain_seconds: float + shutdown_timeout_seconds: int + # Lab4: attachments. Metadata lives in PostgreSQL, bytes in a private + # S3 bucket; the API uses its own restricted credentials (not root). + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + s3_object_prefix: str + s3_connect_timeout_seconds: float + s3_read_timeout_seconds: float + # Deadline of one whole S3 call (DNS + retries included), below the LB + # "timeout server" (10 s): a stopped S3 container makes Docker DNS hang + # ~8 s per lookup, which connect_timeout does not cover. + s3_operation_timeout_seconds: float + # PUT gets operation timeout + size / this rate: 10 MiB under load is + # slower than a HEAD. HAProxy allows 40 s for an upload POST. + s3_put_min_bytes_per_second: int + max_upload_bytes: int + max_image_pixels: int + # One upload may not take unlimited RAM/time: the body is spooled to a + # temporary file (only upload_spool_memory_bytes stay in RAM), receiving + # it is limited by upload_timeout_seconds, and at most + # max_concurrent_uploads uploads per replica run at once. + upload_spool_memory_bytes: int + upload_timeout_seconds: float + max_concurrent_uploads: int + upload_slot_wait_seconds: float + # Janitor: an upload intent older than upload_grace_seconds means the + # request died between S3 PUT and COMMIT; its object is deleted. + upload_grace_seconds: int + janitor_interval_seconds: float + # Experiments only (see REPORT.md): "" | crash_after_s3_put | db_error_after_s3_put + fault_injection: str + + +def load_settings() -> Settings: + session_store = os.getenv("SESSION_STORE", "postgres").lower() + + if session_store not in {"postgres", "valkey", "redis"}: + raise RuntimeError("SESSION_STORE must be postgres, valkey or redis.") + + cache_url = None + if session_store != "postgres": + cache_url = read_value("CACHE_URL") + + return Settings( + app_host=os.getenv("APP_HOST", "0.0.0.0"), + app_port=int(os.getenv("APP_PORT", "8080")), + database_url=read_value("DATABASE_URL"), + session_ttl_seconds=read_positive_int("SESSION_TTL_SECONDS", "3600"), + db_pool_size=read_positive_int("DB_POOL_SIZE", "10"), + db_max_overflow=int(os.getenv("DB_MAX_OVERFLOW", "5")), + db_pool_timeout_seconds=read_positive_float("DB_POOL_TIMEOUT_SECONDS", "3"), + db_connect_timeout_seconds=read_positive_float( + "DB_CONNECT_TIMEOUT_SECONDS", "3" + ), + db_statement_timeout_ms=read_positive_int("DB_STATEMENT_TIMEOUT_MS", "5000"), + cache_url=cache_url, + session_cache_ttl_seconds=read_positive_int( + "SESSION_CACHE_TTL_SECONDS", "60" + ), + cache_timeout_seconds=read_positive_float("CACHE_TIMEOUT_SECONDS", "0.25"), + cache_max_connections=read_positive_int("CACHE_MAX_CONNECTIONS", "20"), + cache_retry_after_seconds=read_positive_float( + "CACHE_RETRY_AFTER_SECONDS", "5" + ), + instance_id=os.getenv("INSTANCE_ID") or socket.gethostname(), + idempotency_ttl_seconds=read_positive_int( + "IDEMPOTENCY_TTL_SECONDS", "86400" + ), + shutdown_drain_seconds=read_positive_float( + "SHUTDOWN_DRAIN_SECONDS", "3" + ), + shutdown_timeout_seconds=read_positive_int( + "SHUTDOWN_TIMEOUT_SECONDS", "8" + ), + s3_endpoint=read_value("S3_ENDPOINT", "http://s3:9000"), + s3_region=os.getenv("S3_REGION", "us-east-1"), + s3_bucket=read_value("S3_BUCKET", "messenger-attachments"), + s3_access_key_id=read_value("S3_ACCESS_KEY_ID"), + s3_secret_access_key=read_value("S3_SECRET_ACCESS_KEY"), + s3_object_prefix=os.getenv("S3_OBJECT_PREFIX", "originals/"), + s3_connect_timeout_seconds=read_positive_float( + "S3_CONNECT_TIMEOUT_SECONDS", "2" + ), + s3_read_timeout_seconds=read_positive_float( + "S3_READ_TIMEOUT_SECONDS", "5" + ), + s3_operation_timeout_seconds=read_positive_float( + "S3_OPERATION_TIMEOUT_SECONDS", "6" + ), + s3_put_min_bytes_per_second=read_positive_int( + "S3_PUT_MIN_BYTES_PER_SECOND", "1048576" + ), + max_upload_bytes=read_positive_int("MAX_UPLOAD_BYTES", "10485760"), + max_image_pixels=read_positive_int("MAX_IMAGE_PIXELS", "40000000"), + upload_spool_memory_bytes=read_positive_int( + "UPLOAD_SPOOL_MEMORY_BYTES", "1048576" + ), + upload_timeout_seconds=read_positive_float( + "UPLOAD_TIMEOUT_SECONDS", "60" + ), + max_concurrent_uploads=read_positive_int("MAX_CONCURRENT_UPLOADS", "8"), + upload_slot_wait_seconds=read_positive_float( + "UPLOAD_SLOT_WAIT_SECONDS", "10" + ), + upload_grace_seconds=read_positive_int("UPLOAD_GRACE_SECONDS", "900"), + janitor_interval_seconds=read_positive_float( + "JANITOR_INTERVAL_SECONDS", "60" + ), + fault_injection=os.getenv("FAULT_INJECTION", ""), + ) + + +settings = load_settings() diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..35674fa6 --- /dev/null +++ b/app/db.py @@ -0,0 +1,57 @@ +from collections.abc import AsyncGenerator + +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from app.config import settings + + +database_url = settings.database_url.replace( + "postgresql://", + "postgresql+asyncpg://", + 1, +) + +# Bounded pool: at most db_pool_size + db_max_overflow connections per API +# process. Waiting for a free connection, connecting and every statement are +# limited, so an unavailable database produces a fast 503 instead of a hang. +engine = create_async_engine( + database_url, + pool_pre_ping=True, + pool_size=settings.db_pool_size, + max_overflow=settings.db_max_overflow, + pool_timeout=settings.db_pool_timeout_seconds, + pool_recycle=1800, + connect_args={ + "timeout": settings.db_connect_timeout_seconds, + "command_timeout": settings.db_statement_timeout_ms / 1000 + 1, + "server_settings": { + "statement_timeout": str(settings.db_statement_timeout_ms), + "idle_in_transaction_session_timeout": "10000", + "application_name": "messenger-api", + }, + }, +) + +SessionLocal = async_sessionmaker( + bind=engine, + expire_on_commit=False, + autoflush=False, +) + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + async with SessionLocal() as session: + yield session + + +async def check_database_ready() -> bool: + try: + async with engine.connect() as connection: + await connection.exec_driver_sql("SELECT 1") + return True + except Exception: + return False diff --git a/app/deps.py b/app/deps.py new file mode 100644 index 00000000..2fe50ea1 --- /dev/null +++ b/app/deps.py @@ -0,0 +1,61 @@ +""" +FastAPI dependencies for Lab2: +- Bearer authentication; +- pagination query parameters. +""" + +from typing import Optional + +from fastapi import Depends, Query, Request, Security +from fastapi.security import HTTPAuthorizationCredentials, HTTPBasic, HTTPBearer +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import get_db +from app.errors import Unauthorized +from app.services import current_user_from_bearer_token + + +# Security schemes make the "Authorize" button in /docs send real +# Authorization headers. Errors keep the contract format: auto_error=False +# and our own 401 responses. +bearer_scheme = HTTPBearer( + scheme_name="Bearer", + auto_error=False, + description="Opaque session token from POST /api/v1/auth/sessions.", +) + + +class BasicForDocs(HTTPBasic): + """Documents Basic auth; the header is parsed by app.security.""" + + async def __call__(self, request: Request) -> None: # type: ignore[override] + return None + + +basic_scheme = BasicForDocs( + scheme_name="Basic", + auto_error=False, + description="Only for POST /api/v1/auth/sessions.", +) + + +async def require_user( + credentials: Optional[HTTPAuthorizationCredentials] = Security( + bearer_scheme + ), + db: AsyncSession = Depends(get_db), +) -> dict: + if credentials is None or not credentials.credentials.strip(): + raise Unauthorized("Bearer") + + return await current_user_from_bearer_token( + db, + credentials.credentials.strip(), + ) + + +def pagination_params( + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> tuple[int, Optional[str]]: + return limit, cursor \ No newline at end of file diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..4e57091d --- /dev/null +++ b/app/errors.py @@ -0,0 +1,130 @@ +import re +from fastapi import Request +from fastapi.responses import JSONResponse +from uuid import uuid4 + +from app.config import settings +from app.lifecycle import draining + + +class ApiError(Exception): + def __init__( + self, + status_code: int, + code: str, + message: str, + headers: dict[str, str] | None = None, + ) -> None: + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +class InvalidRequest(ApiError): + def __init__(self, message: str = "Invalid request.") -> None: + super().__init__(400, "invalid_request", message) + + +class Unauthorized(ApiError): + def __init__(self, scheme: str = "Bearer") -> None: + if scheme == "Basic": + challenge = 'Basic realm="messenger", charset="UTF-8"' + else: + challenge = 'Bearer realm="messenger"' + + super().__init__( + 401, + "unauthorized", + "Authentication is required.", + {"WWW-Authenticate": challenge}, + ) + + +class NotFound(ApiError): + def __init__(self, message: str = "Object not found.") -> None: + super().__init__(404, "not_found", message) + + +class Conflict(ApiError): + def __init__(self, message: str = "Conflict.") -> None: + super().__init__(409, "conflict", message) + + +class Unavailable(ApiError): + def __init__(self) -> None: + super().__init__(503, "unavailable", "Service is unavailable.") + + +class PayloadTooLarge(ApiError): + def __init__(self, message: str = "Payload is too large.") -> None: + super().__init__(413, "payload_too_large", message) + + +class UnsupportedMediaType(ApiError): + def __init__( + self, + message: str = "Only PNG and JPEG images are accepted.", + ) -> None: + super().__init__(415, "unsupported_media_type", message) + + +class RateLimited(ApiError): + def __init__(self, message: str, retry_after_seconds: int = 1) -> None: + super().__init__( + 429, + "rate_limited", + message, + {"Retry-After": str(retry_after_seconds)}, + ) + +# X-Request-Id from HAProxy (or the client) is reused only if it is short and +# printable; anything else is replaced, so it cannot inject into logs. +REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def incoming_request_id(request: Request) -> str: + value = request.headers.get("X-Request-Id", "") + if REQUEST_ID_PATTERN.fullmatch(value): + return value + return str(uuid4()) + + +def request_id(request: Request) -> str: + return getattr(request.state, "request_id", str(uuid4())) + + +def common_headers(request: Request) -> dict[str, str]: + """Headers of every API response, including errors (lab3: X-Instance-Id).""" + headers = { + "X-Request-Id": request_id(request), + "X-Instance-Id": settings.instance_id, + } + if draining.is_set(): + # Ask HAProxy not to reuse the connection to a replica that is stopping. + headers["Connection"] = "close" + return headers + + +async def api_error_handler( + request: Request, + exc: ApiError, +) -> JSONResponse: + rid = request_id(request) + + headers = { + **common_headers(request), + **exc.headers, + } + + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": exc.code, + "message": exc.message, + "request_id": rid, + } + }, + headers=headers, + ) \ No newline at end of file diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..1e337088 --- /dev/null +++ b/app/images.py @@ -0,0 +1,264 @@ +"""Content-based validation of uploaded PNG/JPEG images. + +The declared Content-Type is never trusted: the format is detected from the +first bytes and must match it. Dimensions are read from the header and the +pixel limit is applied BEFORE any decoding, so a small "pixel bomb" file +cannot make the API allocate gigabytes. + +- PNG is validated by our own streaming reader: chunk lengths, CRC of every + chunk, IHDR, and the IDAT zlib stream is decompressed in 64 KiB pieces + that are counted and thrown away. Memory stays O(64 KiB) for any image. +- JPEG: Pillow reads the header (lazy open); then the image is decoded with + draft() at 1/8 scale, which makes libjpeg read the whole entropy-coded + stream (a truncated/corrupted file raises) while allocating only 1/64 of + the full-size pixel buffer. Large progressive JPEGs keep all DCT + coefficients in memory even with draft(), so above + PROGRESSIVE_DECODE_MAX_PIXELS only the marker structure is checked. + +The functions are synchronous (CPU work); call them via asyncio.to_thread. +""" +from __future__ import annotations + +import struct +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image + +# Our own limit is applied before Pillow decodes anything. +Image.MAX_IMAGE_PIXELS = None + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" +CHUNK = 64 * 1024 +PROGRESSIVE_DECODE_MAX_PIXELS = 16_000_000 + + +class InvalidImage(Exception): + """Not a PNG/JPEG, a corrupted file or a Content-Type mismatch (-> 415).""" + + +class TooManyPixels(Exception): + """Decoded image would exceed MAX_IMAGE_PIXELS (-> 413).""" + + +@dataclass(frozen=True) +class ImageInfo: + content_type: str + width: int + height: int + + +def sniff_content_type(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return "image/png" + if head.startswith(JPEG_SIGNATURE): + return "image/jpeg" + return None + + +def validate_image( + file: BinaryIO, + declared_content_type: str, + max_pixels: int, +) -> ImageInfo: + file.seek(0) + detected = sniff_content_type(file.read(len(PNG_SIGNATURE))) + file.seek(0) + + if detected is None: + raise InvalidImage("Content is not a PNG or JPEG image.") + + if detected != declared_content_type: + raise InvalidImage("Content-Type does not match the image content.") + + if detected == "image/png": + width, height = _validate_png(file, max_pixels) + else: + width, height = _validate_jpeg(file, max_pixels) + + return ImageInfo(detected, width, height) + + +# --- PNG ------------------------------------------------------------------- + +# color type -> (channels, allowed bit depths) +_PNG_COLOR_TYPES = { + 0: (1, {1, 2, 4, 8, 16}), + 2: (3, {8, 16}), + 3: (1, {1, 2, 4, 8}), + 4: (2, {8, 16}), + 6: (4, {8, 16}), +} +_ADAM7_PASSES = ( + (0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), + (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2), +) + + +def _read_exact(file: BinaryIO, size: int) -> bytes: + data = file.read(size) + if len(data) != size: + raise InvalidImage("Image file is truncated.") + return data + + +def _png_raw_size(width: int, height: int, bits_per_pixel: int, interlace: int) -> int: + """Size of the decompressed IDAT stream: filter byte + packed row bytes.""" + def scanlines(w: int, h: int) -> int: + if w == 0 or h == 0: + return 0 + return h * (1 + (w * bits_per_pixel + 7) // 8) + + if interlace == 0: + return scanlines(width, height) + + total = 0 + for x0, y0, dx, dy in _ADAM7_PASSES: + pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0 + pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0 + total += scanlines(pass_width, pass_height) + return total + + +def _validate_png(file: BinaryIO, max_pixels: int) -> tuple[int, int]: + _read_exact(file, len(PNG_SIGNATURE)) + + width = height = 0 + expected_raw = 0 + produced_raw = 0 + inflater = zlib.decompressobj() + color_type = -1 + seen_plte = False + idat_state = "before" # before -> inside -> after + first = True + + while True: + header = _read_exact(file, 8) + length, chunk_type = struct.unpack(">I4s", header) + + if length > 0x7FFFFFFF: + raise InvalidImage("Invalid PNG chunk length.") + + if first and chunk_type != b"IHDR": + raise InvalidImage("PNG must start with IHDR.") + + crc = zlib.crc32(chunk_type) + is_idat = chunk_type == b"IDAT" + + if is_idat: + if idat_state == "after": + raise InvalidImage("PNG IDAT chunks are not consecutive.") + if color_type == 3 and not seen_plte: + raise InvalidImage("PNG palette image without PLTE.") + idat_state = "inside" + elif idat_state == "inside": + idat_state = "after" + + data = b"" + remaining = length + while remaining: + piece = _read_exact(file, min(CHUNK, remaining)) + remaining -= len(piece) + crc = zlib.crc32(piece, crc) + + if is_idat: + produced_raw += _inflate_counting( + inflater, piece, expected_raw - produced_raw + ) + elif chunk_type == b"IHDR": + data += piece + + (stored_crc,) = struct.unpack(">I", _read_exact(file, 4)) + if stored_crc != crc: + raise InvalidImage("PNG chunk CRC mismatch.") + + if first: + first = False + if length != 13: + raise InvalidImage("Invalid PNG IHDR.") + (width, height, bit_depth, color_type, + compression, filter_method, interlace) = struct.unpack(">IIBBBBB", data) + + if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise InvalidImage("Invalid PNG dimensions.") + if width * height > max_pixels: + # Before a single IDAT byte is decompressed. + raise TooManyPixels() + if color_type not in _PNG_COLOR_TYPES: + raise InvalidImage("Invalid PNG color type.") + channels, depths = _PNG_COLOR_TYPES[color_type] + if bit_depth not in depths or compression != 0 or filter_method != 0 \ + or interlace not in (0, 1): + raise InvalidImage("Invalid PNG IHDR.") + expected_raw = _png_raw_size(width, height, channels * bit_depth, interlace) + elif chunk_type == b"PLTE": + seen_plte = True + elif chunk_type == b"IEND": + break + + if idat_state == "before": + raise InvalidImage("PNG has no image data.") + + produced_raw += len(inflater.flush()) + if not inflater.eof or produced_raw != expected_raw: + raise InvalidImage("PNG image data is corrupted or truncated.") + + return width, height + + +def _inflate_counting(inflater, data: bytes, allowed: int) -> int: + """Decompress and discard; return the number of produced bytes.""" + produced = 0 + try: + while data: + if inflater.eof: + raise InvalidImage("PNG has data after the end of the zlib stream.") + out = inflater.decompress(data, CHUNK) + produced += len(out) + if produced > allowed: + raise InvalidImage("PNG image data is larger than IHDR declares.") + data = inflater.unconsumed_tail + except zlib.error: + raise InvalidImage("PNG image data is corrupted.") from None + return produced + + +# --- JPEG ------------------------------------------------------------------ + +def _validate_jpeg(file: BinaryIO, max_pixels: int) -> tuple[int, int]: + try: + with Image.open(file, formats=["JPEG"]) as image: + width, height = image.size + + if width < 1 or height < 1: + raise InvalidImage("Invalid JPEG dimensions.") + if width * height > max_pixels: + # Header only: no pixel has been decoded yet. + raise TooManyPixels() + + progressive = bool( + image.info.get("progressive") or image.info.get("progression") + ) + + if progressive and width * height > PROGRESSIVE_DECODE_MAX_PIXELS: + _check_jpeg_end(file) + else: + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + except (InvalidImage, TooManyPixels): + raise + except Exception: + # Pillow raises OSError/SyntaxError/ValueError/... for broken data. + raise InvalidImage("JPEG image is corrupted or truncated.") from None + + return width, height + + +def _check_jpeg_end(file: BinaryIO) -> None: + file.seek(0, 2) + size = file.tell() + file.seek(max(0, size - 2)) + if file.read(2) != b"\xff\xd9": + raise InvalidImage("JPEG image is truncated.") diff --git a/app/lifecycle.py b/app/lifecycle.py new file mode 100644 index 00000000..87d43274 --- /dev/null +++ b/app/lifecycle.py @@ -0,0 +1,6 @@ +"""Process lifecycle state shared by the server and request handlers.""" +import threading + +# Set on the first SIGTERM/SIGINT. While draining, /health/ready answers 503 +# and responses carry "Connection: close", but requests are still served. +draining = threading.Event() diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..a1e2efd7 --- /dev/null +++ b/app/main.py @@ -0,0 +1,460 @@ +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from uuid import UUID + +from typing import Literal + +from fastapi import Depends, FastAPI, Header, Query, Request, Response, Security +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from sqlalchemy import exc as sa_exc +from starlette.exceptions import HTTPException as StarletteHTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from app.attachments import ( + get_attachment, + janitor_loop, + open_attachment_content, + upload_attachment, +) +from app.cache import session_cache +from app.db import check_database_ready, engine, get_db +from app.deps import basic_scheme, pagination_params, require_user +from app.errors import ( + ApiError, + InvalidRequest, + Unavailable, + api_error_handler, + common_headers, + incoming_request_id, +) +from app.lifecycle import draining +from app.models import ( + Attachment, + Chat, + ChatPage, + CreateChat, + CreateMessage, + CreateUser, + Health, + Message, + MessagePage, + Session, + User, +) +from app.storage import storage +from app.services import ( + create_chat, + create_message, + create_user_session, + get_chat, + list_chats, + list_messages, + logout_current_session, + register_user, +) + + +@asynccontextmanager +async def lifespan(_app: FastAPI) -> AsyncIterator[None]: + await storage.start() + janitor = asyncio.create_task(janitor_loop()) + yield + janitor.cancel() + await asyncio.gather(janitor, return_exceptions=True) + await storage.close() + await session_cache.close() + await engine.dispose() + + +app = FastAPI( + title="MTUSI Messenger — лабораторная 4", + version="1.4.0", + lifespan=lifespan, +) + +@app.middleware("http") +async def add_request_id(request: Request, call_next): + request.state.request_id = incoming_request_id(request) + + response = await call_next(request) + response.headers.update(common_headers(request)) + return response + +@app.exception_handler(ApiError) +async def handle_api_error( + request: Request, + exc: Exception, +) -> JSONResponse: + if not isinstance(exc, ApiError): + raise exc + + return await api_error_handler(request, exc) + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler( + request: Request, + exc: StarletteHTTPException, +) -> JSONResponse: + # Unknown routes/methods: the contract error format instead of + # FastAPI's default {"detail": ...}. + if exc.status_code == 404: + error = ApiError(404, "not_found", "Route not found.") + else: + error = ApiError(exc.status_code, "invalid_request", "Invalid request.") + + return await api_error_handler(request, error) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler( + request: Request, + _exc: RequestValidationError, +) -> JSONResponse: + return JSONResponse( + status_code=400, + content={ + "error": { + "code": "invalid_request", + "message": "Invalid request.", + "request_id": common_headers(request)["X-Request-Id"], + } + }, + headers=common_headers(request), + ) + + +logger = logging.getLogger("messenger.api") + + +def is_dependency_failure(exc: Exception) -> bool: + """PostgreSQL is unreachable, restarting, overloaded or timed out.""" + if isinstance( + exc, + ( + sa_exc.OperationalError, + sa_exc.InterfaceError, + sa_exc.TimeoutError, # connection pool exhausted + OSError, + TimeoutError, + ), + ): + return True + + return isinstance(exc, sa_exc.DBAPIError) and exc.connection_invalidated + + +async def dependency_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + if not is_dependency_failure(exc): + return await unexpected_error_handler(request, exc) + + logger.warning("database unavailable: %s", type(exc).__name__) + return await api_error_handler(request, Unavailable()) + + +for _exc_class in (sa_exc.DBAPIError, sa_exc.TimeoutError, OSError, TimeoutError): + app.add_exception_handler(_exc_class, dependency_error_handler) + + +@app.exception_handler(Exception) +async def unexpected_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + headers = common_headers(request) + # Only the type: SQL parameters and headers may contain secrets. + logger.error("unexpected error: %s", type(exc).__name__) + + return JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "Internal server error.", + "request_id": headers["X-Request-Id"], + } + }, + headers=headers, + ) + + +@app.get( + "/health/live", + response_model=Health, +) +async def liveness() -> dict[str, str]: + return {"status": "ok"} + + +@app.get( + "/health/ready", + response_model=Health, +) +async def readiness() -> dict[str, str]: + # A draining replica reports "not ready" so HAProxy stops sending new + # requests to it before the listener is closed. + # S3 is deliberately NOT checked: without it only attachments degrade + # (503), users/chats/messages keep working. If readiness depended on S3, + # an S3 outage would take every replica out of the LB. + if draining.is_set() or not await check_database_ready(): + raise Unavailable() + + return {"status": "ok"} + + +@app.post( + "/api/v1/users", + status_code=201, + response_model=User, +) +async def register_user_route( + payload: CreateUser, + db: AsyncSession = Depends(get_db), +): + return await register_user( + db, + username=payload.username, + display_name=payload.display_name, + password=payload.password, + ) + + +@app.post( + "/api/v1/auth/sessions", + status_code=201, + response_model=Session, +) +async def login( + request: Request, + db: AsyncSession = Depends(get_db), + _basic: None = Security(basic_scheme), +): + # Contract: login takes only "Authorization: Basic", no request body. + # Any body (malformed JSON, "{}", ...) is rejected before the password check. + if await request.body(): + raise InvalidRequest("POST /auth/sessions does not accept a request body.") + + return await create_user_session( + db, + request.headers.get("Authorization"), + ) + + +@app.delete( + "/api/v1/auth/sessions/current", + status_code=204, +) +async def logout( + request: Request, + db: AsyncSession = Depends(get_db), + _current_user: dict = Depends(require_user), +) -> Response: + _, _, token = request.headers["Authorization"].partition(" ") + + await logout_current_session( + db, + token.strip(), + ) + + return Response(status_code=204) + + +@app.get( + "/api/v1/users/me", + response_model=User, +) +async def current_user_route( + current_user: dict = Depends(require_user), +): + return current_user + + +@app.post( + "/api/v1/chats", + status_code=201, + response_model=Chat, +) +async def create_chat_route( + payload: CreateChat, + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + return await create_chat( + db, + title=payload.title, + member_ids=payload.member_ids, + current_user=current_user, + ) + + +@app.get( + "/api/v1/chats", + response_model=ChatPage, +) +async def list_chats_route( + page: tuple[int, str | None] = Depends(pagination_params), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + limit, cursor = page + + return await list_chats( + db, + limit=limit, + cursor=cursor, + current_user=current_user, + ) + + +@app.get( + "/api/v1/chats/{chat_id}", + response_model=Chat, +) +async def get_chat_route( + chat_id: UUID, + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + return await get_chat( + db, + chat_id=chat_id, + current_user=current_user, + ) + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + status_code=201, + response_model=Message, +) +async def create_message_route( + chat_id: UUID, + payload: CreateMessage, + response: Response, + idempotency_key: str | None = Header( + default=None, + alias="Idempotency-Key", + ), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + message, replayed = await create_message( + db, + chat_id=chat_id, + text_value=payload.text, + attachment_ids=payload.attachment_ids, + idempotency_key=idempotency_key, + current_user=current_user, + ) + + if replayed: + response.headers["Idempotent-Replayed"] = "true" + + return message + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +async def list_messages_route( + chat_id: UUID, + page: tuple[int, str | None] = Depends(pagination_params), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + limit, cursor = page + + return await list_messages( + db, + chat_id=chat_id, + limit=limit, + cursor=cursor, + current_user=current_user, + ) + + +UPLOAD_BODY = { + "requestBody": { + "required": True, + "description": "Raw image bytes, not multipart and not JSON/base64.", + "content": { + "image/png": {"schema": {"type": "string", "format": "binary"}}, + "image/jpeg": {"schema": {"type": "string", "format": "binary"}}, + }, + } +} + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + status_code=201, + response_model=Attachment, + openapi_extra=UPLOAD_BODY, +) +async def upload_attachment_route( + chat_id: UUID, + request: Request, + filename: str = Query(min_length=1, max_length=255), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + return await upload_attachment( + request, + db, + chat_id=chat_id, + filename=filename, + current_user=current_user, + ) + + +@app.get( + "/api/v1/attachments/{attachment_id}", + response_model=Attachment, +) +async def get_attachment_route( + attachment_id: UUID, + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + return await get_attachment( + db, + attachment_id=attachment_id, + current_user=current_user, + ) + + +@app.get( + "/api/v1/attachments/{attachment_id}/content", + response_class=StreamingResponse, + responses={200: {"content": {"image/png": {}, "image/jpeg": {}}}}, +) +async def download_attachment_route( + attachment_id: UUID, + # Known variant names are accepted so that a ready attachment without + # that variant answers 404 (contract); any other value is 400. + variant: Literal["original", "thumbnail", "crop", "optimized"] = Query( + default="original" + ), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + chunks, headers = await open_attachment_content( + db, + attachment_id=attachment_id, + variant=variant, + current_user=current_user, + ) + return StreamingResponse( + chunks, + media_type=headers.pop("Content-Type"), + headers=headers, + ) diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..7b01367c --- /dev/null +++ b/app/models.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class ApiModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class CreateUser(ApiModel): + username: str = Field( + min_length=3, + max_length=32, + pattern=r"^[a-z0-9_]{3,32}$", + ) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def username_is_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("username must not contain only spaces.") + return value + + @field_validator("display_name") + @classmethod + def display_name_is_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError( + "display_name must not contain only spaces." + ) + return value + + @field_validator("password") + @classmethod + def password_is_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("password must not contain only spaces.") + return value + + +class User(ApiModel): + id: UUID + username: str + display_name: str + created_at: datetime + + @field_validator("created_at") + @classmethod + def user_time_is_utc(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("created_at must include UTC timezone.") + return value.astimezone(timezone.utc) + + +class CreateChat(ApiModel): + # Contract: 1-100 code points (lab2 allowed 255 by mistake). + title: str = Field(min_length=1, max_length=100) + # Contract: 1-100 members (minItems/maxItems in openapi.json). + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_is_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must not contain only spaces.") + return value + + +class Chat(ApiModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + @field_validator("created_at") + @classmethod + def chat_time_is_utc(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("created_at must include UTC timezone.") + return value.astimezone(timezone.utc) + + +class ChatPage(ApiModel): + items: list[Chat] + next_cursor: str | None = None + + +class CreateMessage(ApiModel): + # Lab4: text may be "" when the message carries attachments. + text: str = Field(max_length=4000) + # 0-10 unique IDs of attachments of the same chat; missing == []. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def attachment_ids_are_unique(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError("attachment_ids must be unique.") + return value + + @model_validator(mode="after") + def text_or_attachments(self) -> "CreateMessage": + if not self.attachment_ids and not self.text.strip(): + raise ValueError( + "text must not be blank when there are no attachments." + ) + return self + + +class Message(ApiModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] = Field(default_factory=list) + + @field_validator("created_at") + @classmethod + def message_time_is_utc(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("created_at must include UTC timezone.") + return value.astimezone(timezone.utc) + + +class MessagePage(ApiModel): + items: list[Message] + next_cursor: str | None = None + + +class Session(ApiModel): + token: str = Field(min_length=1) + token_type: Literal["Bearer"] + expires_at: datetime + user: User + + @field_validator("expires_at") + @classmethod + def expiry_time_is_utc(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("expires_at must include UTC timezone.") + return value.astimezone(timezone.utc) + + +class Variant(ApiModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(ApiModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int + sha256: str + # Lab4: synchronous upload, a stored attachment is always ready. + status: Literal["ready"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + @field_validator("created_at", "updated_at") + @classmethod + def attachment_time_is_utc(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("timestamps must include UTC timezone.") + return value.astimezone(timezone.utc) + + +class Health(ApiModel): + status: Literal["ok"] + + +class ErrorInner(ApiModel): + code: str = Field(min_length=1) + message: str = Field(min_length=1) + request_id: str = Field(min_length=1) + + +class Error(ApiModel): + error: ErrorInner \ No newline at end of file diff --git a/app/repositories.py b/app/repositories.py new file mode 100644 index 00000000..2aa1d664 --- /dev/null +++ b/app/repositories.py @@ -0,0 +1,703 @@ +"""PostgreSQL repository functions for MTUSI Messenger Lab4.""" +from __future__ import annotations + +from collections.abc import Iterable +from datetime import datetime +from typing import Any +from uuid import UUID, uuid4 + +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + + +async def create_user( + db: AsyncSession, + *, + username: str, + display_name: str, + password_hash: str, +) -> dict[str, Any] | None: + user_id = uuid4() + try: + result = await db.execute( + text( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (:id, :username, :display_name, :password_hash) + RETURNING id, username, display_name, created_at + """ + ), + { + "id": user_id, + "username": username, + "display_name": display_name, + "password_hash": password_hash, + }, + ) + await db.commit() + return dict(result.mappings().one()) + except IntegrityError: + await db.rollback() + return None + + +async def get_user_by_username( + db: AsyncSession, + username: str, +) -> dict[str, Any] | None: + result = await db.execute( + text( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = :username + """ + ), + {"username": username}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def get_user_by_id( + db: AsyncSession, + user_id: UUID, +) -> dict[str, Any] | None: + result = await db.execute( + text( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = :user_id + """ + ), + {"user_id": user_id}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def user_ids_that_exist( + db: AsyncSession, + member_ids: Iterable[UUID], +) -> set[UUID]: + values = list(member_ids) + if not values: + return set() + + result = await db.execute( + text("SELECT id FROM users WHERE id = ANY(CAST(:member_ids AS uuid[]))"), + {"member_ids": values}, + ) + return {row.id for row in result} + + +async def create_session( + db: AsyncSession, + *, + user_id: UUID, + token_hash: str, + expires_at: datetime, +) -> None: + await db.execute( + text( + """ + INSERT INTO sessions (id, user_id, token_hash, expires_at) + VALUES (:id, :user_id, :token_hash, :expires_at) + """ + ), + { + "id": uuid4(), + "user_id": user_id, + "token_hash": token_hash, + "expires_at": expires_at, + }, + ) + await db.commit() + + +async def get_active_session_user( + db: AsyncSession, + token_hash: str, +) -> dict[str, Any] | None: + result = await db.execute( + text( + """ + SELECT + s.id AS session_id, + s.token_hash, + s.expires_at, + u.id, + u.username, + u.display_name, + u.created_at + FROM sessions AS s + JOIN users AS u ON u.id = s.user_id + WHERE s.token_hash = :token_hash + AND s.revoked_at IS NULL + AND s.expires_at > now() + """ + ), + {"token_hash": token_hash}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def revoke_active_session( + db: AsyncSession, + token_hash: str, +) -> bool: + result = await db.execute( + text( + """ + UPDATE sessions + SET revoked_at = now() + WHERE token_hash = :token_hash + AND revoked_at IS NULL + AND expires_at > now() + """ + ), + {"token_hash": token_hash}, + ) + await db.commit() + return result.rowcount == 1 + + +async def create_chat_with_members( + db: AsyncSession, + *, + title: str, + member_ids: Iterable[UUID], +) -> dict[str, Any]: + chat_id = uuid4() + members = list(member_ids) + + try: + result = await db.execute( + text( + """ + INSERT INTO chats (id, title) + VALUES (:id, :title) + RETURNING id, title, created_at + """ + ), + {"id": chat_id, "title": title}, + ) + chat = dict(result.mappings().one()) + + await db.execute( + text( + """ + INSERT INTO chat_members (chat_id, user_id) + VALUES (:chat_id, :user_id) + """ + ), + [{"chat_id": chat_id, "user_id": user_id} for user_id in members], + ) + await db.commit() + except Exception: + await db.rollback() + raise + + chat["member_ids"] = members + return chat + + +async def get_accessible_chat( + db: AsyncSession, + *, + chat_id: UUID, + user_id: UUID, +) -> dict[str, Any] | None: + result = await db.execute( + text( + """ + SELECT c.id, c.title, c.created_at + FROM chats AS c + JOIN chat_members AS own_membership + ON own_membership.chat_id = c.id + AND own_membership.user_id = :user_id + WHERE c.id = :chat_id + """ + ), + {"chat_id": chat_id, "user_id": user_id}, + ) + chat_row = result.mappings().first() + if chat_row is None: + return None + + members_result = await db.execute( + text( + """ + SELECT user_id + FROM chat_members + WHERE chat_id = :chat_id + ORDER BY user_id ASC + """ + ), + {"chat_id": chat_id}, + ) + chat = dict(chat_row) + chat["member_ids"] = [row.user_id for row in members_result] + return chat + + +async def list_chats_for_user( + db: AsyncSession, + *, + user_id: UUID, + limit: int, + cursor_created_at: datetime | None, + cursor_id: UUID | None, +) -> list[dict[str, Any]]: + query = """ + SELECT c.id, c.title, c.created_at + FROM chats AS c + JOIN chat_members AS own_membership + ON own_membership.chat_id = c.id + AND own_membership.user_id = :user_id \ + """ + params: dict[str, Any] = { + "user_id": user_id, + "limit_plus_one": limit + 1, + } + + if cursor_created_at is not None and cursor_id is not None: + query += """ + WHERE (c.created_at, c.id) > (:cursor_created_at, :cursor_id) + """ + params["cursor_created_at"] = cursor_created_at + params["cursor_id"] = cursor_id + + query += " ORDER BY c.created_at ASC, c.id ASC LIMIT :limit_plus_one" + + result = await db.execute(text(query), params) + chats = [dict(row) for row in result.mappings().all()] + if not chats: + return [] + + chat_ids = [chat["id"] for chat in chats] + members_result = await db.execute( + text( + """ + SELECT chat_id, user_id + FROM chat_members + WHERE chat_id = ANY(CAST(:chat_ids AS uuid[])) + ORDER BY chat_id ASC, user_id ASC + """ + ), + {"chat_ids": chat_ids}, + ) + + members_by_chat: dict[UUID, list[UUID]] = {chat_id: [] for chat_id in chat_ids} + for row in members_result: + members_by_chat[row.chat_id].append(row.user_id) + + for chat in chats: + chat["member_ids"] = members_by_chat[chat["id"]] + + return chats + + +# Attachment IDs of a message in the order of the request. +MESSAGE_ATTACHMENT_IDS = """ + COALESCE( + (SELECT array_agg(ma.attachment_id ORDER BY ma.position) + FROM message_attachments AS ma + WHERE ma.message_id = m.id), + CAST('{}' AS uuid[]) + ) AS attachment_ids +""" + + +class AttachmentsNotFound(Exception): + """Some attachment_ids do not exist or belong to another chat (-> 404).""" + + +async def create_message_idempotent( + db: AsyncSession, + *, + chat_id: UUID, + sender_id: UUID, + text_value: str, + attachment_ids: list[UUID], + idempotency_key: str, + request_hash: str, + ttl_seconds: int, +) -> tuple[dict[str, Any], bool] | None: + """Insert a message at most once per idempotency key; one transaction. + + 1. INSERT the key row. Its primary key (user, chat, method, key) lives in + the shared PostgreSQL, so a concurrent request with the same key on + ANY replica blocks on this INSERT until the first transaction ends. + 2. If the key was claimed: check that every attachment belongs to this + chat, INSERT the message and its attachment links, link the message + to the key and COMMIT everything together. A failure before COMMIT + (also AttachmentsNotFound) rolls back all of it, so an error is never + stored as the final answer and a retry with the same key runs again. + 3. If the key already exists (committed by someone else): return the + stored message, or None when the body differs (-> 409). + + A key older than ttl_seconds is reclaimed by the ON CONFLICT ... WHERE. + Returns (message, replayed) or None on a body mismatch. + """ + key_params = { + "user_id": sender_id, + "chat_id": chat_id, + "key": idempotency_key, + } + + try: + claimed = await db.execute( + text( + """ + INSERT INTO idempotency_keys AS k + (user_id, chat_id, method, key, request_hash) + VALUES (:user_id, :chat_id, 'POST', :key, :request_hash) + ON CONFLICT (user_id, chat_id, method, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, + message_id = NULL, + created_at = now() + WHERE k.created_at < now() - make_interval(secs => :ttl) + RETURNING 1 + """ + ), + {**key_params, "request_hash": request_hash, "ttl": ttl_seconds}, + ) + + if claimed.first() is not None: + if attachment_ids: + # Unknown and foreign IDs look the same (404): the response + # does not reveal that an attachment of another chat exists. + found = await db.execute( + text( + """ + SELECT id FROM attachments + WHERE chat_id = :chat_id + AND id = ANY(CAST(:ids AS uuid[])) + FOR SHARE + """ + ), + {"chat_id": chat_id, "ids": attachment_ids}, + ) + if {row.id for row in found} != set(attachment_ids): + raise AttachmentsNotFound() + + result = await db.execute( + text( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES (:id, :chat_id, :sender_id, :text) + RETURNING id, chat_id, sender_id, text, created_at + """ + ), + { + "id": uuid4(), + "chat_id": chat_id, + "sender_id": sender_id, + "text": text_value, + }, + ) + message = dict(result.mappings().one()) + + if attachment_ids: + await db.execute( + text( + """ + INSERT INTO message_attachments + (message_id, attachment_id, position) + VALUES (:message_id, :attachment_id, :position) + """ + ), + [ + { + "message_id": message["id"], + "attachment_id": attachment_id, + "position": position, + } + for position, attachment_id in enumerate(attachment_ids) + ], + ) + + await db.execute( + text( + """ + UPDATE idempotency_keys + SET message_id = :message_id + WHERE user_id = :user_id + AND chat_id = :chat_id + AND method = 'POST' + AND key = :key + """ + ), + {**key_params, "message_id": message["id"]}, + ) + await db.commit() + message["attachment_ids"] = list(attachment_ids) + return message, False + + stored = await db.execute( + text( + f""" + SELECT k.request_hash, + m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {MESSAGE_ATTACHMENT_IDS} + FROM idempotency_keys AS k + JOIN messages AS m ON m.id = k.message_id + WHERE k.user_id = :user_id + AND k.chat_id = :chat_id + AND k.method = 'POST' + AND k.key = :key + """ + ), + key_params, + ) + row = stored.mappings().first() + await db.commit() + except Exception: + await db.rollback() + raise + + if row is None: + # Not reachable while the key row and the message commit together. + raise RuntimeError("idempotency key without a message") + + message = dict(row) + if message.pop("request_hash") != request_hash: + return None + + return message, True + + +async def list_messages_keyset( + db: AsyncSession, + *, + chat_id: UUID, + limit: int, + cursor_created_at: datetime | None, + cursor_id: UUID | None, +) -> list[dict[str, Any]]: + query = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {MESSAGE_ATTACHMENT_IDS} + FROM messages AS m + WHERE m.chat_id = :chat_id + """ + params: dict[str, Any] = { + "chat_id": chat_id, + "limit_plus_one": limit + 1, + } + + if cursor_created_at is not None and cursor_id is not None: + query += """ + AND (m.created_at, m.id) > (:cursor_created_at, :cursor_id) + """ + params["cursor_created_at"] = cursor_created_at + params["cursor_id"] = cursor_id + + query += " ORDER BY m.created_at ASC, m.id ASC LIMIT :limit_plus_one" + result = await db.execute(text(query), params) + return [dict(row) for row in result.mappings().all()] + + +# --- Attachments ------------------------------------------------------------ + +ATTACHMENT_COLUMNS = """ + a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, + a.sha256, a.status, a.error_code, a.object_key, a.created_at, a.updated_at +""" + + +async def create_upload_intent( + db: AsyncSession, + *, + object_key: str, + attachment_id: UUID, + chat_id: UUID, +) -> None: + """Committed BEFORE the S3 PUT: from now on the object can be found and + cleaned up even if this process dies right after the PUT.""" + try: + await db.execute( + text( + """ + INSERT INTO upload_intents (object_key, attachment_id, chat_id) + VALUES (:object_key, :attachment_id, :chat_id) + """ + ), + { + "object_key": object_key, + "attachment_id": attachment_id, + "chat_id": chat_id, + }, + ) + await db.commit() + except Exception: + await db.rollback() + raise + + +async def finalize_attachment( + db: AsyncSession, + *, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + object_key: str, +) -> dict[str, Any] | None: + """Delete the intent and insert the ready attachment in ONE transaction. + + Returns None if the intent is already gone: the janitor decided this + upload was abandoned and deleted the object, so the row must not be + created. The DELETE waits on the janitor's row lock, so the two can + never both succeed. + """ + try: + claimed = await db.execute( + text( + "DELETE FROM upload_intents WHERE object_key = :object_key RETURNING 1" + ), + {"object_key": object_key}, + ) + if claimed.first() is None: + await db.rollback() + return None + + result = await db.execute( + text( + f""" + INSERT INTO attachments AS a + (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key) + VALUES (:id, :chat_id, :owner_id, :filename, :content_type, + :size_bytes, :sha256, :width, :height, :object_key) + RETURNING {ATTACHMENT_COLUMNS} + """ + ), + { + "id": attachment_id, + "chat_id": chat_id, + "owner_id": owner_id, + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "sha256": sha256, + "width": width, + "height": height, + "object_key": object_key, + }, + ) + row = dict(result.mappings().one()) + await db.commit() + return row + except Exception: + await db.rollback() + raise + + +async def delete_upload_intent(db: AsyncSession, object_key: str) -> None: + try: + await db.execute( + text("DELETE FROM upload_intents WHERE object_key = :object_key"), + {"object_key": object_key}, + ) + await db.commit() + except Exception: + await db.rollback() + raise + + +async def get_accessible_attachment( + db: AsyncSession, + *, + attachment_id: UUID, + user_id: UUID, +) -> dict[str, Any] | None: + """Only members of the attachment's chat see it; others get None (404).""" + result = await db.execute( + text( + f""" + SELECT {ATTACHMENT_COLUMNS} + FROM attachments AS a + JOIN chat_members AS cm + ON cm.chat_id = a.chat_id + AND cm.user_id = :user_id + WHERE a.id = :attachment_id + """ + ), + {"attachment_id": attachment_id, "user_id": user_id}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def lock_upload_intent( + db: AsyncSession, + object_key: str, +) -> dict[str, Any] | None: + """Lock the intent of one upload (waits for a finishing transaction); + the transaction stays open. None: no intent, nothing to compensate.""" + result = await db.execute( + text( + """ + SELECT i.object_key, + EXISTS (SELECT 1 FROM attachments AS a + WHERE a.object_key = i.object_key) AS committed + FROM upload_intents AS i + WHERE i.object_key = :object_key + FOR UPDATE OF i + """ + ), + {"object_key": object_key}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def lock_stale_intent( + db: AsyncSession, + grace_seconds: int, +) -> dict[str, Any] | None: + """Lock ONE intent older than the grace period; the transaction stays open. + + SKIP LOCKED: an upload that is finishing right now holds the row lock in + finalize_attachment and is skipped; several replicas can run the janitor + at once without touching the same intent. + """ + result = await db.execute( + text( + """ + SELECT i.object_key, + EXISTS (SELECT 1 FROM attachments AS a + WHERE a.object_key = i.object_key) AS committed + FROM upload_intents AS i + WHERE i.created_at < now() - make_interval(secs => :grace) + ORDER BY i.created_at + LIMIT 1 + FOR UPDATE OF i SKIP LOCKED + """ + ), + {"grace": grace_seconds}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def attachment_keys(db: AsyncSession) -> set[str]: + result = await db.execute(text("SELECT object_key FROM attachments")) + return {row.object_key for row in result} + + +async def intent_keys(db: AsyncSession) -> dict[str, datetime]: + result = await db.execute( + text("SELECT object_key, created_at FROM upload_intents") + ) + return {row.object_key: row.created_at for row in result} diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..c7c63954 --- /dev/null +++ b/app/security.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import secrets +from datetime import datetime, timezone +from uuid import UUID + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + + +password_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return password_hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return password_hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError): + return False + + +def generate_session_token() -> str: + return secrets.token_urlsafe(32) + + +def hash_session_token(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def parse_basic_authorization(value: str | None) -> tuple[str, str] | None: + if value is None or not value.startswith("Basic "): + return None + + try: + encoded_credentials = value[len("Basic "):] + decoded = base64.b64decode( + encoded_credentials, + validate=True, + ).decode("utf-8") + + username, password = decoded.split(":", 1) + except ( + ValueError, + UnicodeDecodeError, + binascii.Error, + ): + return None + + if not username or not username.isascii(): + return None + + return username, password + + +def _encode_cursor_payload(payload: dict[str, str]) -> str: + raw = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def _decode_cursor_payload(cursor: str) -> dict[str, str] | None: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + payload = json.loads(raw.decode("utf-8")) + + if not isinstance(payload, dict): + return None + + return payload + except ( + ValueError, + TypeError, + UnicodeDecodeError, + binascii.Error, + ): + return None + + +def _utc_timestamp(value: datetime) -> str: + if value.tzinfo is None: + raise ValueError("Cursor datetime must be timezone-aware.") + + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def encode_chat_cursor( + created_at: datetime, + chat_id: UUID, +) -> str: + return _encode_cursor_payload( + { + "kind": "chat", + "created_at": _utc_timestamp(created_at), + "id": str(chat_id), + } + ) + + +def decode_chat_cursor( + cursor: str, +) -> tuple[datetime, UUID] | None: + payload = _decode_cursor_payload(cursor) + + if payload is None or payload.get("kind") != "chat": + return None + + try: + created_at = datetime.fromisoformat( + payload["created_at"].replace("Z", "+00:00") + ) + chat_id = UUID(payload["id"]) + except (KeyError, ValueError, AttributeError): + return None + + return created_at, chat_id + + +def encode_message_cursor( + chat_id: UUID, + created_at: datetime, + message_id: UUID, +) -> str: + return _encode_cursor_payload( + { + "kind": "message", + "chat_id": str(chat_id), + "created_at": _utc_timestamp(created_at), + "id": str(message_id), + } + ) + + +def decode_message_cursor( + cursor: str, +) -> tuple[UUID, datetime, UUID] | None: + payload = _decode_cursor_payload(cursor) + + if payload is None or payload.get("kind") != "message": + return None + + try: + chat_id = UUID(payload["chat_id"]) + created_at = datetime.fromisoformat( + payload["created_at"].replace("Z", "+00:00") + ) + message_id = UUID(payload["id"]) + except (KeyError, ValueError, AttributeError): + return None + + return chat_id, created_at, message_id \ No newline at end of file diff --git a/app/server.py b/app/server.py new file mode 100644 index 00000000..5a87d3f4 --- /dev/null +++ b/app/server.py @@ -0,0 +1,70 @@ +"""Entry point: uvicorn with a drain phase before the graceful shutdown. + +Plain uvicorn stops accepting connections immediately on SIGTERM. Behind +HAProxy that turns every stop into a burst of refused connections until the +active health check notices. Instead, on the first signal: + + t = 0 readiness -> 503, keep serving (Connection: close); + t = drain (3 s) HAProxy has marked the server DOWN (inter 1 s x fall 2), + uvicorn closes the listener and waits for in-flight + requests up to SHUTDOWN_TIMEOUT_SECONDS (8 s). + +stop_grace_period in compose.yaml (15 s) is larger than 3 + 8 s. +A second signal skips the drain phase. +""" +import logging +import threading + +import uvicorn + +from app.config import settings +from app.lifecycle import draining + + +logger = logging.getLogger("messenger.server") + + +class DrainingServer(uvicorn.Server): + def handle_exit(self, sig, frame) -> None: + if draining.is_set(): + super().handle_exit(sig, frame) + return + + draining.set() + logger.warning( + "signal %s: draining for %.1f s before shutdown", + sig, + settings.shutdown_drain_seconds, + ) + # Set should_exit directly instead of super().handle_exit(): uvicorn + # would re-raise the captured SIGTERM after shutdown and the process + # would exit with 143 instead of a clean 0. + timer = threading.Timer( + settings.shutdown_drain_seconds, + self._begin_shutdown, + ) + timer.daemon = True + timer.start() + + def _begin_shutdown(self) -> None: + logger.warning("drain finished: closing listener") + self.should_exit = True + + +def main() -> None: + config = uvicorn.Config( + "app.main:app", + host=settings.app_host, + port=settings.app_port, + proxy_headers=False, + # Longer than HAProxy "timeout http-keep-alive 10s": the LB closes an + # idle connection first, so it never reuses one the API just closed. + timeout_keep_alive=30, + timeout_graceful_shutdown=settings.shutdown_timeout_seconds, + log_level="info", + ) + DrainingServer(config).run() + + +if __name__ == "__main__": + main() diff --git a/app/services.py b/app/services.py new file mode 100644 index 00000000..8d293801 --- /dev/null +++ b/app/services.py @@ -0,0 +1,430 @@ +"""Business rules for MTUSI Messenger Lab4 (users, chats, messages).""" +from __future__ import annotations + +import asyncio +import hashlib +import json +import re + +from datetime import datetime, timedelta, timezone +from typing import Any +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app import repositories +from app.cache import REVOKED_SESSION, session_cache +from app.config import settings +from app.errors import ApiError +from app.security import ( + decode_chat_cursor, + decode_message_cursor, + encode_chat_cursor, + encode_message_cursor, + generate_session_token, + hash_password, + hash_session_token, + parse_basic_authorization, + verify_password, +) + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _to_user(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "username": row["username"], + "display_name": row["display_name"], + "created_at": row["created_at"], + } + + +def _to_chat(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": row["created_at"], + } + + +def _to_message(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "chat_id": row["chat_id"], + "sender_id": row["sender_id"], + "text": row["text"], + "created_at": row["created_at"], + "attachment_ids": list(row.get("attachment_ids") or []), + } + + +def _bearer_unauthorized() -> ApiError: + return ApiError( + 401, + "unauthorized", + "Authentication is required.", + headers={ + "WWW-Authenticate": 'Bearer realm="messenger"', + }, + ) + + +def _basic_unauthorized() -> ApiError: + return ApiError( + 401, + "unauthorized", + "Invalid credentials.", + headers={ + "WWW-Authenticate": ( + 'Basic realm="messenger", charset="UTF-8"' + ), + }, + ) + + +# Lab3 contract: ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def _invalid_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def _not_found() -> ApiError: + return ApiError(404, "not_found", "Object not found.") + + +async def register_user( + db: AsyncSession, + *, + username: str, + display_name: str, + password: str, +) -> dict[str, Any]: + # Argon2 is CPU-bound (~0.5-1 s); off the event loop so /health/ready keeps answering. + password_hash = await asyncio.to_thread(hash_password, password) + user = await repositories.create_user( + db, + username=username, + display_name=display_name, + password_hash=password_hash, + ) + + if user is None: + raise ApiError( + 409, + "conflict", + "Username is already taken.", + ) + + return _to_user(user) + + +async def create_user_session( + db: AsyncSession, + authorization: str | None, +) -> dict[str, Any]: + credentials = parse_basic_authorization(authorization) + + if credentials is None: + raise _basic_unauthorized() + + username, password = credentials + user = await repositories.get_user_by_username(db, username) + + if user is None or not await asyncio.to_thread( + verify_password, + user["password_hash"], + password, + ): + raise _basic_unauthorized() + + token = generate_session_token() + expires_at = _utc_now() + timedelta( + seconds=settings.session_ttl_seconds + ) + + await repositories.create_session( + db, + user_id=user["id"], + token_hash=hash_session_token(token), + expires_at=expires_at, + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": _to_user(user), + } + + +async def current_user_from_bearer_token( + db: AsyncSession, + token: str, +) -> dict[str, Any]: + token_hash = hash_session_token(token) + cached = await session_cache.get(token_hash) + + if cached is REVOKED_SESSION: + raise _bearer_unauthorized() + + if cached is not None: + return cached + + # Cache miss or cache unavailable: PostgreSQL decides. + session_user = await repositories.get_active_session_user(db, token_hash) + + if session_user is None: + raise _bearer_unauthorized() + + user = _to_user(session_user) + await session_cache.put(token_hash, user, session_user["expires_at"]) + return user + + +async def logout_current_session( + db: AsyncSession, + token: str, +) -> None: + token_hash = hash_session_token(token) + revoked = await repositories.revoke_active_session(db, token_hash) + + if not revoked: + raise _bearer_unauthorized() + + # Invalidate after COMMIT in the source of truth. + await session_cache.revoke(token_hash) + + +async def create_chat( + db: AsyncSession, + *, + title: str, + member_ids: list[UUID], + current_user: dict[str, Any], +) -> dict[str, Any]: + unique_member_ids = set(member_ids) + + if len(unique_member_ids) != len(member_ids): + raise _invalid_request( + "member_ids must not contain duplicates." + ) + + if current_user["id"] not in unique_member_ids: + raise _invalid_request( + "The current user must be included in member_ids." + ) + + existing_ids = await repositories.user_ids_that_exist( + db, + unique_member_ids, + ) + + if existing_ids != unique_member_ids: + raise _invalid_request( + "One or more member_ids do not exist." + ) + + chat = await repositories.create_chat_with_members( + db, + title=title, + member_ids=member_ids, + ) + + return _to_chat(chat) + + +async def get_chat( + db: AsyncSession, + *, + chat_id: UUID, + current_user: dict[str, Any], +) -> dict[str, Any]: + chat = await repositories.get_accessible_chat( + db, + chat_id=chat_id, + user_id=current_user["id"], + ) + + if chat is None: + raise _not_found() + + return _to_chat(chat) + + +async def list_chats( + db: AsyncSession, + *, + limit: int, + cursor: str | None, + current_user: dict[str, Any], +) -> dict[str, Any]: + cursor_created_at: datetime | None = None + cursor_id: UUID | None = None + + if cursor is not None: + decoded = decode_chat_cursor(cursor) + + if decoded is None: + raise _invalid_request("Invalid cursor.") + + cursor_created_at, cursor_id = decoded + + rows = await repositories.list_chats_for_user( + db, + user_id=current_user["id"], + limit=limit, + cursor_created_at=cursor_created_at, + cursor_id=cursor_id, + ) + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = None + if has_more and page: + last = page[-1] + next_cursor = encode_chat_cursor( + last["created_at"], + last["id"], + ) + + return { + "items": [_to_chat(row) for row in page], + "next_cursor": next_cursor, + } + + +def _request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str: + """Hash of the validated body: key order and whitespace do not matter, + a missing attachment_ids equals []. Without attachments the canonical + form is the lab3 one, so keys stored by lab3 still replay correctly.""" + body: dict[str, Any] = {"text": text_value} + if attachment_ids: + body["attachment_ids"] = [str(item) for item in attachment_ids] + canonical = json.dumps( + body, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +async def create_message( + db: AsyncSession, + *, + chat_id: UUID, + text_value: str, + attachment_ids: list[UUID], + idempotency_key: str | None, + current_user: dict[str, Any], +) -> tuple[dict[str, Any], bool]: + """Send a message once per (user, POST, chat, Idempotency-Key). + + Returns (message, replayed). A replay re-checks the session (done by + require_user) and chat membership before returning the stored result. + """ + if idempotency_key is None or not IDEMPOTENCY_KEY_PATTERN.fullmatch( + idempotency_key + ): + raise _invalid_request( + "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}." + ) + + chat = await repositories.get_accessible_chat( + db, + chat_id=chat_id, + user_id=current_user["id"], + ) + + if chat is None: + raise _not_found() + + try: + result = await repositories.create_message_idempotent( + db, + chat_id=chat_id, + sender_id=current_user["id"], + text_value=text_value, + attachment_ids=attachment_ids, + idempotency_key=idempotency_key, + request_hash=_request_fingerprint(text_value, attachment_ids), + ttl_seconds=settings.idempotency_ttl_seconds, + ) + except repositories.AttachmentsNotFound: + raise _not_found() from None + + if result is None: + raise ApiError( + 409, + "conflict", + "Idempotency-Key was already used with a different body.", + ) + + message, replayed = result + return _to_message(message), replayed + + +async def list_messages( + db: AsyncSession, + *, + chat_id: UUID, + limit: int, + cursor: str | None, + current_user: dict[str, Any], +) -> dict[str, Any]: + chat = await repositories.get_accessible_chat( + db, + chat_id=chat_id, + user_id=current_user["id"], + ) + + if chat is None: + raise _not_found() + + cursor_created_at: datetime | None = None + cursor_id: UUID | None = None + + if cursor is not None: + decoded = decode_message_cursor(cursor) + + if decoded is None: + raise _invalid_request("Invalid cursor.") + + cursor_chat_id, cursor_created_at, cursor_id = decoded + + if cursor_chat_id != chat_id: + raise _invalid_request( + "Cursor belongs to another chat." + ) + + rows = await repositories.list_messages_keyset( + db, + chat_id=chat_id, + limit=limit, + cursor_created_at=cursor_created_at, + cursor_id=cursor_id, + ) + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = None + if has_more and page: + last = page[-1] + next_cursor = encode_message_cursor( + chat_id, + last["created_at"], + last["id"], + ) + + return { + "items": [_to_message(row) for row in page], + "next_cursor": next_cursor, + } \ No newline at end of file diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..18a457b6 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,172 @@ +"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway). + +One long-lived aiobotocore client per API process, path-style addressing, +short connect/read timeouts, at most 2 attempts and an overall deadline +S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a +fast 503 on the attachment endpoints only. Any client-side or +network error is raised as StorageUnavailable; a missing key is +ObjectMissing. Object keys are generated by the server (see services.py); +the user's filename never becomes part of a key. +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import Any, BinaryIO + +from aiobotocore.config import AioConfig +from aiobotocore.session import get_session +from botocore.exceptions import BotoCoreError, ClientError + +from app.config import settings + + +logger = logging.getLogger("messenger.storage") + +DOWNLOAD_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 is unreachable, timed out or refused the request.""" + + +class ObjectMissing(Exception): + """The key does not exist in the bucket.""" + + +@dataclass +class StoredObject: + key: str + size: int + last_modified: Any + + +class ObjectStorage: + def __init__(self) -> None: + self._stack: AsyncExitStack | None = None + self._client = None + self.bucket = settings.s3_bucket + + async def start(self) -> None: + self._stack = AsyncExitStack() + session = get_session() + self._client = await self._stack.enter_async_context( + session.create_client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id, + aws_secret_access_key=settings.s3_secret_access_key, + config=AioConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + retries={"max_attempts": 2, "mode": "standard"}, + max_pool_connections=settings.max_concurrent_uploads * 2 + 4, + ), + ) + ) + + async def close(self) -> None: + if self._stack is not None: + await self._stack.aclose() + self._stack = None + self._client = None + + @staticmethod + def _deadline(size: int = 0): + return asyncio.timeout( + settings.s3_operation_timeout_seconds + + size / settings.s3_put_min_bytes_per_second + ) + + def _require_client(self): + if self._client is None: + raise StorageUnavailable("S3 client is not started") + return self._client + + async def put( + self, + key: str, + body: BinaryIO, + size: int, + content_type: str, + ) -> None: + """Upload a seekable file. SigV4 signs the payload SHA-256, so S3 + itself rejects bytes that were damaged on the way.""" + client = self._require_client() + body.seek(0) + try: + async with self._deadline(size): + await client.put_object( + Bucket=self.bucket, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ) + except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]: + """GET the object; S3 errors surface here, before any byte is sent.""" + client = self._require_client() + try: + async with self._deadline(): + response = await client.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: + raise ObjectMissing(key) from exc + raise StorageUnavailable(type(exc).__name__) from exc + except (BotoCoreError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + body = response["Body"] + + async def chunks() -> AsyncIterator[bytes]: + try: + async for chunk in body.iter_chunks(DOWNLOAD_CHUNK): + yield chunk + finally: + body.close() + + return chunks(), int(response["ContentLength"]) + + async def delete(self, key: str) -> None: + """Idempotent: deleting a missing key succeeds in S3.""" + client = self._require_client() + try: + async with self._deadline(): + await client.delete_object(Bucket=self.bucket, Key=key) + except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + async def exists(self, key: str) -> bool: + client = self._require_client() + try: + async with self._deadline(): + await client.head_object(Bucket=self.bucket, Key=key) + return True + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}: + return False + raise StorageUnavailable(type(exc).__name__) from exc + except (BotoCoreError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]: + client = self._require_client() + paginator = client.get_paginator("list_objects_v2") + try: + async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): + for item in page.get("Contents", []): + yield StoredObject(item["Key"], item["Size"], item["LastModified"]) + except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + +storage = ObjectStorage() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..a5b64c95 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,277 @@ +#file: noinspection ComposeUnknownValues +# Lab4: client -> HAProxy -> API x N -> PostgreSQL (source of truth: data, +# sessions, idempotency keys, attachment metadata) + Valkey (session cache) +# + SeaweedFS S3 (private bucket with attachment bytes). +# Start: see REPORT.md, "Запуск из чистого clone". +name: messenger-lab4 + +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}" + SESSION_STORE: valkey + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_CACHE_TTL_SECONDS: "60" + CACHE_TIMEOUT_SECONDS: "0.25" + CACHE_MAX_CONNECTIONS: "20" + CACHE_RETRY_AFTER_SECONDS: "5" + DB_POOL_SIZE: "10" + DB_MAX_OVERFLOW: "5" + DB_POOL_TIMEOUT_SECONDS: "3" + DB_CONNECT_TIMEOUT_SECONDS: "3" + DB_STATEMENT_TIMEOUT_MS: "5000" + IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}" + SHUTDOWN_DRAIN_SECONDS: "3" + SHUTDOWN_TIMEOUT_SECONDS: "8" + # Lab4: S3 with the API's own restricted credentials (not admin). + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_OBJECT_PREFIX: originals/ + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: "2" + S3_READ_TIMEOUT_SECONDS: "5" + S3_OPERATION_TIMEOUT_SECONDS: "6" + S3_PUT_MIN_BYTES_PER_SECOND: "1048576" + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + UPLOAD_SPOOL_MEMORY_BYTES: "1048576" + UPLOAD_TIMEOUT_SECONDS: "60" + MAX_CONCURRENT_UPLOADS: "8" + UPLOAD_SLOT_WAIT_SECONDS: "10" + UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}" + JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}" + # Experiments only: crash_after_s3_put | db_error_after_s3_put (REPORT.md). + FAULT_INJECTION: "${FAULT_INJECTION:-}" + +services: + db: + image: postgres:16.4-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + networks: [backend] + stop_grace_period: 30s + restart: unless-stopped + deploy: + resources: + limits: { cpus: "1.0", memory: 512M } + + # Session cache only (lab2). PostgreSQL stays the source of truth, so the + # cache is intentionally not persisted. It is shared by all API replicas: + # a logout on one replica writes the "revoked" tombstone seen by all. + valkey: + image: valkey/valkey:8.0.1-alpine + command: + - sh + - -c + - >- + exec valkey-server + --requirepass "$$(cat /run/secrets/valkey_password)" + --save "" + --appendonly no + --maxmemory 64mb + --maxmemory-policy allkeys-lru + secrets: [valkey_password] + healthcheck: + test: + - CMD-SHELL + - valkey-cli -a "$$(cat /run/secrets/valkey_password)" --no-auth-warning ping | grep -q PONG + interval: 2s + timeout: 3s + retries: 20 + networks: [backend] + restart: unless-stopped + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + + # S3: SeaweedFS all-in-one. No "ports": reachable only from the internal + # backend network; its admin web UIs are bound to 127.0.0.1 inside the + # container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume. + s3: + image: chrislusf/seaweedfs:3.97 + entrypoint: ["/bin/sh", "/opt/lab/start.sh"] + environment: + # Soft limit of the Go heap: without it the GC lets SeaweedFS grow to + # the container limit under parallel 10 MiB PUTs (measured, REPORT.md). + GOMEMLIMIT: 600MiB + volumes: + - s3_data:/data + - ./infra/seaweedfs:/opt/lab:ro + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_access_key_id + - s3_secret_access_key + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 60s + networks: [backend] + stop_grace_period: 30s + restart: unless-stopped + deploy: + resources: + # 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed + # SeaweedFS to the limit and the kernel killed it (REPORT.md). + limits: { cpus: "1.0", memory: 1G } + + # Idempotent bucket bootstrap + self-checks (private bucket, least + # privilege of the API key). The only container with the S3 admin key. + s3-init: + build: . + image: messenger-lab4-api:${APP_VERSION:-dev} + command: ["python", "scripts/s3_bootstrap.py"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_OBJECT_PREFIX: originals/ + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3: { condition: service_healthy } + networks: [backend] + restart: "no" + deploy: + resources: + limits: { cpus: "0.5", memory: 128M } + + # Runs the migrations once, before any API replica starts (no race of + # replicas). A repeated run only prints SKIP. + migrate: + build: . + image: messenger-lab4-api:${APP_VERSION:-dev} + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + db: { condition: service_healthy } + networks: [backend] + restart: "no" + deploy: + resources: + limits: { cpus: "0.5", memory: 128M } + + api: + build: . + image: messenger-lab4-api:${APP_VERSION:-dev} + environment: *api-env + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + # No "ports": replicas are reachable only through the LB. + # No "container_name": the service is scaled by deploy.replicas / --scale. + networks: [edge, backend] + depends_on: + db: { condition: service_healthy } + valkey: { condition: service_healthy } + migrate: { condition: service_completed_successfully } + s3-init: { condition: service_completed_successfully } + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)" + interval: 2s + timeout: 3s + retries: 2 + start_period: 5s + stop_signal: SIGTERM + # > SHUTDOWN_DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (3 + 8 s). + stop_grace_period: 15s + restart: unless-stopped + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: { cpus: "0.5", memory: 256M } + + lb: + image: haproxy:3.0.11-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "${LB_PORT:-8080}:8080" + # Backend state page for the operator, only on the host loopback. + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + networks: [edge] + depends_on: + api: { condition: service_healthy } + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 2s + retries: 3 + restart: unless-stopped + deploy: + resources: + limits: { cpus: "0.5", memory: 128M } + + # Runs the public and own checks in Linux (needed on Windows: the hooks are + # POSIX .sh scripts that call the docker CLI). Profile "tools": not started + # by a plain "up". The Docker socket grants control over the Docker host — + # use it only on the lab stand. + tester: + profiles: [tools] + build: + context: . + dockerfile: student-tests/Dockerfile.tester + working_dir: /work + environment: + BASE_URL: http://lb:8080 + COMPOSE_PROJECT: messenger-lab4 + S3_ENDPOINT: http://s3:9000 + volumes: + - ./:/work + - /var/run/docker.sock:/var/run/docker.sock + # backend: only for the "private bucket from outside the API" check. + networks: [edge, backend] + +networks: + edge: {} + # PostgreSQL, Valkey and S3: no route outside, the LB is not attached. + backend: + internal: true + +volumes: + postgres_data: {} + s3_data: {} + +secrets: + postgres_password: + file: ./infra/secrets/postgres_password.txt + database_url: + file: ./infra/secrets/database_url.txt + valkey_password: + file: ./infra/secrets/valkey_password.txt + cache_url: + file: ./infra/secrets/cache_url.txt + s3_admin_access_key_id: + file: ./infra/secrets/s3_admin_access_key_id.txt + s3_admin_secret_access_key: + file: ./infra/secrets/s3_admin_secret_access_key.txt + s3_access_key_id: + file: ./infra/secrets/s3_access_key_id.txt + s3_secret_access_key: + file: ./infra/secrets/s3_secret_access_key.txt diff --git a/evidence/api-no-local-files.txt b/evidence/api-no-local-files.txt new file mode 100644 index 00000000..398734c3 --- /dev/null +++ b/evidence/api-no-local-files.txt @@ -0,0 +1,14 @@ +## f00b790a44e9 +mounts=[{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_access_key_id.txt","Destination":"/run/secrets/s3_access_key_id","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_secret_access_key.txt","Destination":"/run/secrets/s3_secret_access_key","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\database_url.txt","Destination":"/run/secrets/database_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\cache_url.txt","Destination":"/run/secrets/cache_url","Mode":"","RW":false,"Propagation":"rprivate"}] readonly_rootfs=false +tmp files: +total 8 +drwxrwxrwt 1 root root 4096 Oct 30 11:44 . +drwxr-xr-x 1 root root 4096 Oct 30 11:44 .. +files > 10KB written by appuser outside /proc,/sys: +## bb5edb449d7c +mounts=[{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\database_url.txt","Destination":"/run/secrets/database_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\cache_url.txt","Destination":"/run/secrets/cache_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_access_key_id.txt","Destination":"/run/secrets/s3_access_key_id","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_secret_access_key.txt","Destination":"/run/secrets/s3_secret_access_key","Mode":"","RW":false,"Propagation":"rprivate"}] readonly_rootfs=false +tmp files: +total 8 +drwxrwxrwt 2 root root 4096 Dec 2 2024 . +drwxr-xr-x 1 root root 4096 Oct 30 11:44 .. +files > 10KB written by appuser outside /proc,/sys: diff --git a/evidence/attachment-edge-cases.txt b/evidence/attachment-edge-cases.txt new file mode 100644 index 00000000..5e2e84b0 --- /dev/null +++ b/evidence/attachment-edge-cases.txt @@ -0,0 +1,63 @@ + Container messenger-lab4-tester-run-cb4971c6a617 Creating + Container messenger-lab4-tester-run-cb4971c6a617 Created +OK JPEG fixture -> 201 ready: 201 (ожидается 201 ) +OK участник скачивает JPEG побайтно, без redirect: 200 (ожидается 200 ) +OK PNG ровно 10 MiB -> 201: 201 (ожидается 201 ) +OK 10 MiB скачан побайтно: 200 (ожидается 200 ) +OK PNG ровно 40 млн пикселей -> 201: 201 (ожидается 201 ) +OK 10 MiB + 1 байт (Content-Length) -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK 11 MiB chunked без Content-Length -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK PNG-«бомба» 50 млн пикселей (~10 KB файл) -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK PNG 2^31-1 x 2^31-1 в IHDR -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK пустое тело -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK PNG обрезан пополам -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG с неверным CRC -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG с мусором в zlib при верном CRC -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG без IEND -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK JPEG обрезан -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK JPEG только сигнатура -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK GIF как image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK случайные байты как image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK HTML как image/jpeg -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG под Content-Type image/jpeg -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK JPEG под Content-Type image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG под text/plain -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG под image/gif -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK multipart/form-data -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK без Content-Type -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK filename ../../etc/passwd -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename C:\boot.png -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename NUL в имени -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename перевод строки -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename ESC-последовательность -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename bidi override (gnp.exe) -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename только пробелы -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename .. -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename 256 символов -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename пустой filename -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename нет filename -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename ровно 255 символов -> 201: 201 (ожидается 201 ) +OK upload без токена -> 401: 401 unauthorized (ожидается 401 unauthorized) +OK посторонний upload в чужой чат -> 404: 404 not_found (ожидается 404 not_found) +OK upload в несуществующий чат -> 404: 404 not_found (ожидается 404 not_found) +OK посторонний GET /attachments/id -> 404: 404 not_found (ожидается 404 not_found) +OK неизвестный attachment -> 404: 404 not_found (ожидается 404 not_found) +OK посторонний GET /attachments/id/content -> 404: 404 not_found (ожидается 404 not_found) +OK неизвестный attachment/content -> 404: 404 not_found (ожидается 404 not_found) +OK attachment_id не UUID -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK ready без варианта thumbnail -> 404: 404 not_found (ожидается 404 not_found) +OK variant неизвестного имени -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK вложение другого чата (своего) -> 404: 404 not_found (ожидается 404 not_found) +OK вложение чужого чата -> 404: 404 not_found (ожидается 404 not_found) +OK одно из ID неизвестно -> 404, ничего не создано: 404 not_found (ожидается 404 not_found) +OK дубли attachment_ids -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK 11 attachment_ids -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK attachment_id не UUID -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK пробельный text без вложений -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK Bob: text="" + вложение Alice из общего чата -> 201: 201 (ожидается 201 ) +OK повтор Idempotency-Key -> то же сообщение: 201 (ожидается 201 ) +OK тот же ключ, другое тело -> 409: 409 conflict (ожидается 409 conflict) +OK история: ровно одно сообщение с вложением: 200 (ожидается 200 ) +OK после оборванной загрузки API работает: 201 (ожидается 201 ) +OK ошибка в общем формате с request_id: 415 unsupported_media_type (ожидается 415 ) +PASS: граничные случаи вложений соответствуют контракту diff --git a/evidence/bootstrap-idempotent.txt b/evidence/bootstrap-idempotent.txt new file mode 100644 index 00000000..0430cdfa --- /dev/null +++ b/evidence/bootstrap-idempotent.txt @@ -0,0 +1,30 @@ +## s3-init run 1 +keep bucket messenger-attachments +PASS api credentials: PUT object +PASS api credentials: GET object +PASS anonymous GET object -> 403 +PASS anonymous LIST bucket -> 403 +PASS anonymous PUT object -> 403 +PASS api credentials: CreateBucket denied +PASS api credentials: DeleteBucket denied +PASS api credentials: DELETE object +PASS api credentials: ListBuckets shows no foreign buckets +PASS: bucket bootstrap complete (idempotent, private, least privilege) +## s3-init run 2 +keep bucket messenger-attachments +PASS api credentials: PUT object +PASS api credentials: GET object +PASS anonymous GET object -> 403 +PASS anonymous LIST bucket -> 403 +PASS anonymous PUT object -> 403 +PASS api credentials: CreateBucket denied +PASS api credentials: DeleteBucket denied +PASS api credentials: DELETE object +PASS api credentials: ListBuckets shows no foreign buckets +PASS: bucket bootstrap complete (idempotent, private, least privilege) +## migrate rerun +SKIP 001_initial_schema.sql +SKIP 002_indexes.sql +SKIP 003_idempotency_keys.sql +SKIP 004_attachments.sql +PASS: migrations applied diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..5e20948e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,24 @@ +## run 1 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 3.972} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +messenger-lab4-api-2 bb5edb449d7c Up 41 seconds (healthy) +messenger-lab4-api-1 f00b790a44e9 Exited (0) 12 seconds ago +messenger-lab3-api-6 82d4e8380bcc Exited (0) 6 days ago +messenger-lab3-api-5 9478eeefc0a7 Exited (0) 6 days ago +mtusi2026lab2-api-1 01469cb1e76e Exited (137) 2 weeks ago +started api container f00b790a44e9 +running api replicas: 2 +## run 2 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 4.208} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +messenger-lab4-api-2 bb5edb449d7c Up About a minute (healthy) +messenger-lab4-api-1 f00b790a44e9 Exited (0) 11 seconds ago +messenger-lab3-api-6 82d4e8380bcc Exited (0) 6 days ago +messenger-lab3-api-5 9478eeefc0a7 Exited (0) 6 days ago +mtusi2026lab2-api-1 01469cb1e76e Exited (137) 2 weeks ago +started api container f00b790a44e9 +running api replicas: 2 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..f3b27960 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,22 @@ +## before +messenger-lab4-api-1 f00b790a44e9 started=2026-10-30T11:44:20.074794065Z status=running health=healthy +messenger-lab4-api-2 bb5edb449d7c started=2026-10-30T11:44:19.748733251Z status=running health=healthy +messenger-lab4-db-1 331396b7fb10 started=2026-10-30T11:09:07.431846589Z status=running health=healthy +messenger-lab4-lb-1 ffe0343f6dc2 started=2026-10-30T11:23:41.657935704Z status=running health=healthy +messenger-lab4-s3-1 16bae2ec7298 started=2026-10-30T11:45:54.317998359Z status=running health=healthy +messenger-lab4-valkey-1 b7f2d1a478a8 started=2026-10-30T11:09:07.702076066Z status=running health=healthy +volume messenger-lab4_postgres_data +volume messenger-lab4_s3_data +## test +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +## after +messenger-lab4-api-1 f00b790a44e9 started=2026-10-30T11:47:37.860335522Z status=running health=healthy +messenger-lab4-api-2 bb5edb449d7c started=2026-10-30T11:47:37.593545231Z status=running health=healthy +messenger-lab4-db-1 331396b7fb10 started=2026-10-30T11:47:11.170703341Z status=running health=healthy +messenger-lab4-lb-1 ffe0343f6dc2 started=2026-10-30T11:47:44.439408787Z status=running health=starting +messenger-lab4-s3-1 16bae2ec7298 started=2026-10-30T11:47:11.637306219Z status=running health=healthy +messenger-lab4-valkey-1 b7f2d1a478a8 started=2026-10-30T11:47:11.434151225Z status=running health=healthy +volume messenger-lab4_postgres_data +volume messenger-lab4_s3_data diff --git a/evidence/s3-failures.txt b/evidence/s3-failures.txt new file mode 100644 index 00000000..1a78f061 --- /dev/null +++ b/evidence/s3-failures.txt @@ -0,0 +1,34 @@ + Container messenger-lab4-tester-run-cabae88c67e1 Creating + Container messenger-lab4-tester-run-cabae88c67e1 Created +OK object_key есть в PostgreSQL и не совпадает с filename/ID +OK API не раскрывает object_key +OK анонимный GET /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный HEAD /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный GET /messenger-attachments?list-type=2 напрямую в S3 -> 403 (403) +OK анонимный GET /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный PUT /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный DELETE /messenger-attachments/ напрямую в S3 -> 403 (403) +OK объект не изменён анонимными PUT/DELETE +OK S3 не опубликован на хосте (docker port пуст) '' +OK LB не проксирует путь S3 (404) +OK SeaweedFS master UI/API :9333 недоступен из сети стенда +OK SeaweedFS filer UI/API :8888 недоступен из сети стенда +OK SeaweedFS volume :8080 недоступен из сети стенда +OK SeaweedFS master gRPC :19333 недоступен из сети стенда +OK SeaweedFS filer gRPC :18888 недоступен из сети стенда +OK SeaweedFS admin :23646 недоступен из сети стенда +-- s3 остановлен +OK /health/ready при отказе S3 -> 200 (S3 не критичен для чата) (200) +OK сообщение со ссылкой на готовое вложение -> 201 (201) +OK metadata из PostgreSQL -> 200 (200) +OK upload -> 503 unavailable (503 unavailable, 6.13 s) +OK upload отказал быстрее timeout LB (< 8 s) (6.13 s) +OK download -> 503 unavailable (503 unavailable, 6.01 s) +OK download отказал быстрее timeout LB (< 8 s) (6.01 s) +OK пользовательская ошибка при отказе S3 остаётся 415 (415) + upload_intents после неудачных upload: 2 +-- s3 запущен +OK s3 снова healthy +OK после восстановления S3 upload -> 201 (201) +OK старый файл скачивается побайтно (200) +PASS: bucket приватен вне API, отказ S3 даёт 503 только для вложений diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/upload-compensation.txt b/evidence/upload-compensation.txt new file mode 100644 index 00000000..4efbb8b1 --- /dev/null +++ b/evidence/upload-compensation.txt @@ -0,0 +1,11 @@ +OK db_error_after_s3_put: 503 (503) +OK db_error_after_s3_put: компенсация убрала intent +OK crash_after_s3_put: клиент не получил 201 (RemoteDisconnected) +OK crash_after_s3_put: остались intent и объект +OK janitor до grace ничего не удаляет +-- ждём grace 21 s +OK janitor после grace удалил intent от crash (cleaned=1) +OK janitor удалил объект от crash +OK свежий intent не тронут +OK готовое вложение цело +PASS: компенсация и janitor не оставляют вечных объектов diff --git a/evidence/upload-failover.txt b/evidence/upload-failover.txt new file mode 100644 index 00000000..39ec4fcc --- /dev/null +++ b/evidence/upload-failover.txt @@ -0,0 +1,3 @@ +OK upload через bb5edb449d7c -> bb5edb449d7c остановлена -> download через ['f00b790a44e9']: SHA-256 совпал +OK upload через f00b790a44e9 -> f00b790a44e9 остановлена -> download через ['bb5edb449d7c']: SHA-256 совпал +PASS: файл доступен после остановки любой обслужившей реплики diff --git a/evidence/upload-memory-c12.json b/evidence/upload-memory-c12.json new file mode 100644 index 00000000..e5c88e5a --- /dev/null +++ b/evidence/upload-memory-c12.json @@ -0,0 +1,74 @@ +{ + "concurrency": 12, + "file_bytes": 9962537, + "statuses": { + "201": 12 + }, + "per_instance": { + "b4841ddecd64": 6, + "dd76ecc96935": 6 + }, + "wall_seconds": 5.18, + "latency_p50_s": 4.98, + "latency_max_s": 5.17, + "sha256_ok": true, + "memory_mib_before": { + "dd76ecc96935": { + "VmHWM": 169.2, + "VmRSS": 105.2, + "peak": 148.3, + "current": 89.0, + "anon": 87.5 + }, + "b4841ddecd64": { + "VmHWM": 169.4, + "VmRSS": 105.6, + "peak": 148.7, + "current": 85.2, + "anon": 83.2 + } + }, + "memory_mib_peak_during": { + "dd76ecc96935": { + "VmRSS_peak": 118.1, + "anon_peak": 108.3, + "samples": 57 + }, + "b4841ddecd64": { + "VmRSS_peak": 119.3, + "anon_peak": 109.5, + "samples": 57 + } + }, + "memory_mib_after": { + "dd76ecc96935": { + "VmHWM": 169.2, + "VmRSS": 118.1, + "peak": 167.7, + "current": 98.5, + "anon": 95.9 + }, + "b4841ddecd64": { + "VmHWM": 169.4, + "VmRSS": 119.3, + "peak": 174.5, + "current": 99.9, + "anon": 97.2 + } + }, + "rss_peak_increase_mib": { + "dd76ecc96935": 12.9, + "b4841ddecd64": 13.7 + }, + "anon_peak_increase_mib": { + "dd76ecc96935": 20.8, + "b4841ddecd64": 26.3 + }, + "s3_server_mib": { + "VmRSS_before": 203.5, + "VmRSS_peak": 546.1, + "anon_peak": 495.1, + "samples": 47 + }, + "naive_in_memory_estimate_mib": 114.0 +} diff --git a/evidence/upload-memory-c20.json b/evidence/upload-memory-c20.json new file mode 100644 index 00000000..ceb95f92 --- /dev/null +++ b/evidence/upload-memory-c20.json @@ -0,0 +1,74 @@ +{ + "concurrency": 20, + "file_bytes": 9962537, + "statuses": { + "201": 20 + }, + "per_instance": { + "dd76ecc96935": 10, + "b4841ddecd64": 10 + }, + "wall_seconds": 9.89, + "latency_p50_s": 8.58, + "latency_max_s": 9.88, + "sha256_ok": true, + "memory_mib_before": { + "dd76ecc96935": { + "VmHWM": 169.4, + "VmRSS": 105.6, + "peak": 148.5, + "current": 85.6, + "anon": 83.1 + }, + "b4841ddecd64": { + "VmHWM": 169.3, + "VmRSS": 105.5, + "peak": 148.8, + "current": 85.8, + "anon": 83.2 + } + }, + "memory_mib_peak_during": { + "dd76ecc96935": { + "VmRSS_peak": 122.1, + "anon_peak": 111.0, + "samples": 98 + }, + "b4841ddecd64": { + "VmRSS_peak": 123.2, + "anon_peak": 113.8, + "samples": 97 + } + }, + "memory_mib_after": { + "dd76ecc96935": { + "VmHWM": 169.4, + "VmRSS": 116.7, + "peak": 195.2, + "current": 97.8, + "anon": 94.5 + }, + "b4841ddecd64": { + "VmHWM": 169.3, + "VmRSS": 123.2, + "peak": 196.0, + "current": 104.3, + "anon": 101.4 + } + }, + "rss_peak_increase_mib": { + "dd76ecc96935": 16.5, + "b4841ddecd64": 17.7 + }, + "anon_peak_increase_mib": { + "dd76ecc96935": 27.9, + "b4841ddecd64": 30.6 + }, + "s3_server_mib": { + "VmRSS_before": 205.3, + "VmRSS_peak": 778.8, + "anon_peak": 698.4, + "samples": 80 + }, + "naive_in_memory_estimate_mib": 190.0 +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..40239779 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,102 @@ +# HAProxy 3.0 — единственная публичная точка входа (порт 8080). +# Backend-адреса берутся из встроенного DNS Docker: имя сервиса `api` возвращает A-запись +# каждого работающего контейнера-реплики. Реплики не публикуют порты наружу. + +global + log stdout format raw local0 info + maxconn 2048 + +# Встроенный DNS Docker. hold valid 2s — как часто перечитывается список реплик +# (новые контейнеры при scale/rolling update появляются в пуле за ≤ 2–3 s). +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold other 2s + hold refused 2s + hold nx 2s + hold timeout 2s + hold obsolete 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + # Таймауты согласованы с приложением: + # connect 1s — недоступная реплика быстро даёт ошибку соединения -> retry на другую; + # server 10s — больше statement_timeout БД (5 s) + ожидания пула (3 s); + # http-keep-alive 10s < timeout_keep_alive uvicorn (30 s): LB закрывает простой первым. + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 5s + retries 2 + option redispatch 1 + +frontend http_in + bind :8080 + # X-Request-Id создаётся на входе (если клиент не прислал корректный), передаётся в API + # и попадает в лог LB — один ID в логах LB и реплики. + unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid + acl has_valid_rid req.hdr(X-Request-Id) -m reg ^[A-Za-z0-9._:-]{1,128}$ + http-request set-header X-Request-Id %[unique-id] unless has_valid_rid + http-request set-var(txn.rid) req.hdr(X-Request-Id) + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.rid)] \"%HM %HPO\"" + default_backend api + + # Ошибки, созданные самим LB (нет живых реплик / timeout), — в формате контракта. + http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]" + http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]" + http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Реплика API не ответила вовремя\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]" + +backend api + # Обычный round-robin без sticky session: корректность обеспечивает общее состояние в PostgreSQL. + balance roundrobin + + # Активная проверка готовности каждые 1 s; 2 неудачи подряд -> DOWN, 2 успеха -> UP. + # Максимальное время обнаружения отказа «процесс жив, но не готов»: + # inter × fall + timeout check = 1 s × 2 + 1 s ≈ 3 s. + # Упавший процесс (connection refused) обнаруживается быстрее: observe layer4 + error-limit 1 + # переводит сервер в DOWN по первой реальной ошибке соединения, а запрос уходит на другую реплику. + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker init-addr none resolve-prefer ipv4 check + timeout check 1s + + # Повторы на другую реплику: + # - ошибка установления соединения повторяется всегда (запрос не дошёл до приложения); + # - L7-повторы (пустой ответ / 503 от дренирующейся реплики) разрешены только для + # безопасных GET и для POST сообщения С Idempotency-Key — его повтор не создаст дубль. + # POST /users, /chats, /auth/sessions после отправки не повторяются (нет ключа идемпотентности). + retry-on conn-failure empty-response 503 + acl is_message_post path_reg ^/api/v1/chats/[^/]+/messages$ + acl has_idem_key req.hdr(Idempotency-Key) -m found + http-request disable-l7-retry if METH_POST !is_message_post + http-request disable-l7-retry if METH_POST !has_idem_key + http-request disable-l7-retry if METH_DELETE + # Lab4: 503 вложения означает отказ общего S3 — на другой реплике тот же S3, повтор + # только утроил бы ожидание клиента (3 × S3_OPERATION_TIMEOUT_SECONDS). conn-failure остаётся. + http-request disable-l7-retry if { path_beg /api/v1/attachments/ } + # Upload: после приёма тела API проверяет PNG/JPEG и делает S3 PUT с дедлайном + # 6 s + size / 1 MiB/s (<= 16 s для 10 MiB) — 10 s «timeout server» мало. + http-request set-timeout server 40s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ } + + # До 8 реплик (scale/rolling update); пустые слоты остаются в MAINT, пока DNS не вернёт адрес. + server-template api 8 api:8080 + +# Служебная страница состояния backend-ов (UP/DOWN, счётчики ошибок/повторов). +# В compose.yaml публикуется только на 127.0.0.1 хоста — не часть публичного API. +frontend stats + bind :8404 + http-request return status 200 content-type text/plain string "ok" if { path /lb-health } + stats enable + stats uri /stats + stats refresh 2s + stats show-legends diff --git a/infra/seaweedfs/s3.json.template b/infra/seaweedfs/s3.json.template new file mode 100644 index 00000000..afdd0046 --- /dev/null +++ b/infra/seaweedfs/s3.json.template @@ -0,0 +1,22 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { "accessKey": "@ADMIN_ACCESS_KEY@", "secretKey": "@ADMIN_SECRET_KEY@" } + ], + "actions": ["Admin", "Read", "List", "Tagging", "Write"] + }, + { + "name": "messenger-api", + "credentials": [ + { "accessKey": "@API_ACCESS_KEY@", "secretKey": "@API_SECRET_KEY@" } + ], + "actions": [ + "Read:messenger-attachments", + "Write:messenger-attachments", + "List:messenger-attachments" + ] + } + ] +} diff --git a/infra/seaweedfs/start.sh b/infra/seaweedfs/start.sh new file mode 100644 index 00000000..1f63d8e3 --- /dev/null +++ b/infra/seaweedfs/start.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# SeaweedFS all-in-one (master + volume + filer + S3 gateway) for the lab stand. +# +# - master/volume/filer and their web UIs listen on 127.0.0.1 INSIDE the +# container only (-ip.bind): their HTTP API has no authentication and must +# not be reachable from the Docker network. Only the S3 gateway (9000) +# listens on all interfaces, and it requires SigV4 credentials. +# - Identities and rights come from s3.json.template; the secret values are +# substituted at start from Docker secrets into a file readable by root +# only (never stored in Git or in the image). +set -eu +umask 077 +s() { cat "/run/secrets/$1"; } +sed -e "s|@ADMIN_ACCESS_KEY@|$(s s3_admin_access_key_id)|" \ + -e "s|@ADMIN_SECRET_KEY@|$(s s3_admin_secret_access_key)|" \ + -e "s|@API_ACCESS_KEY@|$(s s3_access_key_id)|" \ + -e "s|@API_SECRET_KEY@|$(s s3_secret_access_key)|" \ + /opt/lab/s3.json.template > /tmp/s3.json + +exec weed -logtostderr=true server \ + -dir=/data \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ + -master.volumeSizeLimitMB=256 -volume.max=0 \ + -s3 -s3.port=9000 -s3.ip.bind=0.0.0.0 -s3.config=/tmp/s3.json diff --git a/migrations/001_initial_schema.sql b/migrations/001_initial_schema.sql new file mode 100644 index 00000000..64418088 --- /dev/null +++ b/migrations/001_initial_schema.sql @@ -0,0 +1,48 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT users_username_format_check + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + + CONSTRAINT users_display_name_nonblank_check + CHECK (char_length(btrim(display_name)) >= 1) + ); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT chats_title_nonblank_check + CHECK (char_length(btrim(title)) >= 1) + ); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) + ); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT messages_text_nonblank_check + CHECK (char_length(btrim(text)) >= 1) + ); + +CREATE TABLE IF NOT EXISTS sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash CHAR(64) NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); \ No newline at end of file diff --git a/migrations/002_indexes.sql b/migrations/002_indexes.sql new file mode 100644 index 00000000..af47095f --- /dev/null +++ b/migrations/002_indexes.sql @@ -0,0 +1,12 @@ +CREATE INDEX IF NOT EXISTS messages_keyset_idx + ON messages (chat_id, created_at ASC, id ASC); + +CREATE INDEX IF NOT EXISTS chats_keyset_idx + ON chats (created_at ASC, id ASC); + +CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx + ON chat_members (user_id, chat_id); + +CREATE INDEX IF NOT EXISTS sessions_active_token_idx + ON sessions (token_hash, expires_at) + WHERE revoked_at IS NULL; \ No newline at end of file diff --git a/migrations/003_idempotency_keys.sql b/migrations/003_idempotency_keys.sql new file mode 100644 index 00000000..eacde91d --- /dev/null +++ b/migrations/003_idempotency_keys.sql @@ -0,0 +1,18 @@ +-- Lab3: shared idempotency of POST /chats/{chat_id}/messages. +-- Scope: user + method + chat path + key. The key row and the message are +-- written in one transaction, so either both exist or neither does. +-- Expand-only change: the running lab2/lab3 API keeps working with it. +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + method VARCHAR(8) NOT NULL, + key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + message_id UUID REFERENCES messages(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, method, key) +); + +-- For a periodic cleanup of keys older than IDEMPOTENCY_TTL_SECONDS. +CREATE INDEX IF NOT EXISTS idempotency_keys_created_idx + ON idempotency_keys (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..4b9a549e --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,68 @@ +-- Lab4: attachments. PostgreSQL keeps metadata, the bytes live in a private +-- S3 bucket under a server-generated object_key. Expand-only migration: the +-- lab3 API keeps working on this schema. + +-- Text may be empty when the message has attachments; "not blank without +-- attachments" is checked by the API (one message + its links are inserted +-- in one transaction). +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank_check; + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE RESTRICT, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + -- Display name only; never used as a path or an S3 key. + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + -- Lab4 has only "ready": the row is inserted after the object is stored. + status VARCHAR(16) NOT NULL DEFAULT 'ready', + error_code VARCHAR(64), + object_key TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT attachments_content_type_check + CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_check + CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_check + CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status_check + CHECK (status IN ('ready')) +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id); + +-- A message references 0-10 attachments of its own chat; "position" keeps +-- the order of attachment_ids. One attachment may be referenced by several +-- messages of the chat. +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + position SMALLINT NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx + ON message_attachments (attachment_id); + +-- Upload intent: committed BEFORE the S3 PUT and deleted in the same +-- transaction that inserts the attachment row. An intent older than the +-- grace period therefore means "the object may exist in S3, but the request +-- never committed" (crash, lost DB, client gone) -> the janitor deletes the +-- object and then the intent. +CREATE TABLE IF NOT EXISTS upload_intents ( + object_key TEXT PRIMARY KEY, + attachment_id UUID NOT NULL, + chat_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS upload_intents_created_idx + ON upload_intents (created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..efc66e53 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +SQLAlchemy==2.0.36 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +redis==5.0.8 +# Lab4: S3 client (async) and image header/JPEG decoding +aiobotocore==2.15.2 +botocore==1.35.36 +aiohttp==3.10.10 +Pillow==11.0.0 diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100644 index 00000000..d91f686a --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Creates local Docker secrets for a synthetic lab stand. +# Existing files are never overwritten; values are never printed. +set -eu + +dir="$(cd "$(dirname "$0")/.." && pwd)/infra/secrets" +mkdir -p "$dir" +# Host access is limited by the directory mode; the files themselves must stay +# readable by the non-root users inside containers (bind-mounted secrets). +chmod 700 "$dir" 2>/dev/null || true + +random() { + od -An -tx1 -N24 /dev/urandom | tr -d ' \n' +} + +write_new() { + if [ -f "$dir/$1" ]; then + echo "keep infra/secrets/$1" + else + printf '%s' "$2" > "$dir/$1" + chmod 644 "$dir/$1" 2>/dev/null || true + echo "create infra/secrets/$1" + fi +} + +if [ ! -f "$dir/postgres_password.txt" ]; then + write_new postgres_password.txt "$(random)" +fi +pg_password="$(cat "$dir/postgres_password.txt")" +write_new database_url.txt "postgresql://messenger:${pg_password}@db:5432/messenger" + +if [ ! -f "$dir/valkey_password.txt" ]; then + write_new valkey_password.txt "$(random)" +fi +valkey_password="$(cat "$dir/valkey_password.txt")" +write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0" + +# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init +# bootstrap; the API gets its own key pair restricted to one bucket. +write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)" +write_new s3_admin_secret_access_key.txt "$(random)$(random)" +write_new s3_access_key_id.txt "api$(random | cut -c1-17)" +write_new s3_secret_access_key.txt "$(random)$(random)" diff --git a/scripts/kill-one.sh b/scripts/kill-one.sh new file mode 100644 index 00000000..bd30ddc0 --- /dev/null +++ b/scripts/kill-one.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Вариант жёсткого отказа (для сравнения): SIGKILL без graceful shutdown. +# Используется в собственных экспериментах; публичный тест запускается со stop-one.sh. +set -eu +: "${TARGET_INSTANCE_ID:?target API instance alias required}" +. "$(dirname "$0")/lib.sh" +case "$TARGET_INSTANCE_ID" in + *[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;; +esac +for id in $(service_ids api); do + case "$id" in "$TARGET_INSTANCE_ID"*) + docker kill --signal KILL "$id" >/dev/null + echo "killed api container $(short "$id") at $(date -u +%H:%M:%S)" + exit 0 ;; + esac +done +echo "instance $TARGET_INSTANCE_ID is not a running api container" >&2 +exit 3 diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 00000000..0877da12 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Общие функции операторских скриптов. Используют только docker CLI и метки Compose, +# поэтому работают и на хосте, и в контейнере tester (docker.sock). +PROJECT="${COMPOSE_PROJECT:-messenger-lab4}" + +# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные. +service_ids() { + # $1 — сервис, $2 — необязательный флаг -a + docker ps ${2:-} -q --no-trunc \ + --filter "label=com.docker.compose.project=$PROJECT" \ + --filter "label=com.docker.compose.service=$1" +} + +# Ждать, пока все переданные контейнеры станут healthy (timeout в секундах первым аргументом). +# В POSIX sh переменные функций глобальны: имена с префиксом _wh_, чтобы не затереть +# переменные вызывающего цикла. +wait_healthy() { + _wh_timeout="$1"; shift + _wh_deadline=$(( $(date +%s) + _wh_timeout )) + for _wh_id in "$@"; do + while :; do + _wh_state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$_wh_id" 2>/dev/null || echo missing) + case "$_wh_state" in healthy|running) break ;; esac + if [ "$(date +%s)" -ge "$_wh_deadline" ]; then + echo "container $(echo "$_wh_id" | cut -c1-12) not healthy (state=$_wh_state)" >&2 + return 1 + fi + sleep 1 + done + done +} + +short() { echo "$1" | cut -c1-12; } diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..252d602d --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +import asyncpg + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATIONS_DIR = ROOT / "migrations" +# Lab3: migrations run once in the "migrate" service before the API replicas +# start. The advisory lock still serialises two accidental parallel runs +# (e.g. a manual run during a rolling update); the second one then sees all +# versions in schema_migrations and only prints SKIP. +MIGRATION_LOCK_ID = 727003 + + +def read_database_url() -> str: + file_name = os.getenv("DATABASE_URL_FILE") + + if file_name: + return Path(file_name).read_text(encoding="utf-8").strip() + + value = os.getenv("DATABASE_URL") + + if not value: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required.") + + return value + + +def plain_postgres_url(url: str) -> str: + if url.startswith("postgresql+asyncpg://"): + return url.replace("postgresql+asyncpg://", "postgresql://", 1) + + return url + + +async def connect_with_retry(url: str, attempts: int = 30): + for attempt in range(1, attempts + 1): + try: + return await asyncpg.connect(url, timeout=3) + except (OSError, asyncpg.PostgresError) as exc: + if attempt == attempts: + raise + print(f"WAIT database ({type(exc).__name__}), retry {attempt}") + await asyncio.sleep(1) + + +async def migrate() -> None: + connection = await connect_with_retry( + plain_postgres_url(read_database_url()) + ) + + try: + await connection.execute( + "SELECT pg_advisory_lock($1)", + MIGRATION_LOCK_ID, + ) + await connection.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + + applied_rows = await connection.fetch( + "SELECT version FROM schema_migrations" + ) + applied = {row["version"] for row in applied_rows} + + for migration_file in sorted(MIGRATIONS_DIR.glob("*.sql")): + version = migration_file.name + + if version in applied: + print(f"SKIP {version}") + continue + + sql = migration_file.read_text(encoding="utf-8") + + print(f"APPLY {version}") + + async with connection.transaction(): + await connection.execute(sql) + await connection.execute( + "INSERT INTO schema_migrations (version) VALUES ($1)", + version, + ) + + print("PASS: migrations applied") + finally: + await connection.close() + + +if __name__ == "__main__": + asyncio.run(migrate()) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..69df0e94 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Persistence hook: restarts the LB, every API replica and all stores +# (PostgreSQL and S3 with their volumes, Valkey cache). No "down -v", no re-seed. +# Uses only the docker CLI and Compose labels, so it also runs in "tester". +set -eu +. "$(dirname "$0")/lib.sh" +api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db) +valkey=$(service_ids valkey); s3=$(service_ids s3) +[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] && [ -n "$s3" ] || { echo "stack is not running" >&2; exit 2; } +echo "stopping lb + api, then db + valkey + s3" +docker stop --time 15 $lb $api >/dev/null +docker stop --time 30 $db $valkey $s3 >/dev/null +echo "starting db + valkey + s3" +docker start $db $valkey $s3 >/dev/null +wait_healthy 80 $db $valkey $s3 +echo "starting api" +docker start $api >/dev/null +wait_healthy 60 $api +docker start $lb >/dev/null +echo "restart complete: db=$(short $db) valkey=$(short $valkey) s3=$(short $s3) api=$(for i in $api; do short $i; done | tr '\n' ' ')" diff --git a/scripts/s3_bootstrap.py b/scripts/s3_bootstrap.py new file mode 100644 index 00000000..79276d81 --- /dev/null +++ b/scripts/s3_bootstrap.py @@ -0,0 +1,161 @@ +"""Idempotent bootstrap of the private attachments bucket (service s3-init). + +Runs with the S3 ADMIN credentials (only this one-shot container gets them), +before any API replica starts: + + 1. wait until the S3 gateway answers; + 2. create the bucket if it does not exist (a repeated run prints "keep"); + 3. self-checks, the stand does not start if one fails: + - anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private; + - API credentials can PUT/GET/DELETE under the prefix; + - API credentials can NOT create/delete buckets or touch other buckets. + +The API identity and its rights are declared in infra/seaweedfs/s3.json.template. +""" +from __future__ import annotations + +import os +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +import botocore.session +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + + +def secret(name: str) -> str: + file_name = os.getenv(f"{name}_FILE") + if file_name: + return Path(file_name).read_text(encoding="utf-8").strip() + value = os.getenv(name) + if not value: + raise RuntimeError(f"{name} or {name}_FILE is required") + return value + + +ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/") + + +def client(access_key: str, secret_key: str): + return botocore.session.get_session().create_client( + "s3", + endpoint_url=ENDPOINT, + region_name=os.getenv("S3_REGION", "us-east-1"), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 1}, + ), + ) + + +def error_code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "") + + +failures = 0 + + +def check(label: str, ok: bool) -> None: + global failures + failures += not ok + print(("PASS " if ok else "FAIL ") + label, flush=True) + + +def denied(call) -> bool: + try: + call() + except ClientError as exc: + return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"} + return False + + +def anonymous_status(method: str, path: str, data: bytes | None = None) -> int: + request = urllib.request.Request(ENDPOINT + path, data=data, method=method) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY")) + + for attempt in range(1, 61): + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if attempt == 60: + raise + print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True) + time.sleep(1) + + try: + admin.head_bucket(Bucket=BUCKET) + print(f"keep bucket {BUCKET}", flush=True) + except ClientError as exc: + if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}: + raise + try: + admin.create_bucket(Bucket=BUCKET) + print(f"create bucket {BUCKET}", flush=True) + except ClientError as create_exc: + # Two bootstrap runs at the same time: the other one won. + if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + raise + print(f"keep bucket {BUCKET}", flush=True) + + probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}" + + check("api credentials: PUT object", not denied( + lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe"))) + check("api credentials: GET object", + api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe") + + check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403) + check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403) + check("anonymous PUT object -> 403", + anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403) + + other = f"forbidden-{uuid.uuid4().hex[:12]}" + check("api credentials: CreateBucket denied", + denied(lambda: api.create_bucket(Bucket=other))) + check("api credentials: DeleteBucket denied", + denied(lambda: api.delete_bucket(Bucket=BUCKET))) + check("api credentials: DELETE object", not denied( + lambda: api.delete_object(Bucket=BUCKET, Key=probe))) + + def foreign_buckets_hidden() -> bool: + # SeaweedFS denies ListBuckets to a bucket-scoped identity; other + # servers may answer with only the allowed bucket. Both are fine. + try: + names = [b["Name"] for b in api.list_buckets().get("Buckets", [])] + except ClientError as exc: + return error_code(exc) in {"AccessDenied", "403", "Forbidden"} + return all(name == BUCKET for name in names) + + check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden()) + + if failures: + print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True) + return 1 + + print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/start-api.sh b/scripts/start-api.sh new file mode 100644 index 00000000..10bee72c --- /dev/null +++ b/scripts/start-api.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Восстановление после stop-one.sh: запускает остановленные API-реплики и ждёт healthy. +set -eu +. "$(dirname "$0")/lib.sh" +all=$(service_ids api -a) +running=$(service_ids api) +started="" +for id in $all; do + case " $running " in *" $id "*) continue ;; esac + docker start "$id" >/dev/null + started="$started $id" + echo "started api container $(short "$id")" +done +[ -z "$started" ] || wait_healthy 60 $started +echo "running api replicas: $(service_ids api | wc -l | tr -d ' ')" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..533d1e6c --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Failover hook: останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной. +# TARGET_INSTANCE_ID = X-Instance-Id = hostname контейнера = префикс его Docker ID. +# БД, LB и volumes не затрагиваются. Восстановление: scripts/start-api.sh. +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +. "$(dirname "$0")/lib.sh" +case "$TARGET_INSTANCE_ID" in + *[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;; +esac +target="" +for id in $(service_ids api); do + case "$id" in "$TARGET_INSTANCE_ID"*) target="$id" ;; esac +done +if [ -z "$target" ]; then + echo "instance $TARGET_INSTANCE_ID is not a running api container of project $PROJECT" >&2 + exit 3 +fi +# docker stop = SIGTERM -> graceful drain (~3 s) -> выход; SIGKILL только после 15 s. +docker stop --time 15 "$target" >/dev/null +echo "stopped api container $(short "$target") at $(date -u +%H:%M:%S)" diff --git a/student-tests/Dockerfile.tester b/student-tests/Dockerfile.tester new file mode 100644 index 00000000..9d9d000f --- /dev/null +++ b/student-tests/Dockerfile.tester @@ -0,0 +1,6 @@ +# Окружение для запуска проверок: Python 3.12 (stdlib-тесты), make, docker CLI (action-скрипты). +FROM docker:28.5.1-cli AS docker-cli +FROM python:3.12.11-alpine3.22 +RUN apk add --no-cache make +COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker +WORKDIR /work diff --git a/student-tests/attachment_edge_cases.py b/student-tests/attachment_edge_cases.py new file mode 100644 index 00000000..4e2344b6 --- /dev/null +++ b/student-tests/attachment_edge_cases.py @@ -0,0 +1,327 @@ +"""Негативные и граничные сценарии вложений lab4 (только stdlib). + +Проверяет, что пользовательская ошибка никогда не становится 500: + - размер: ровно 10 MiB -> 201; 10 MiB + 1 по Content-Length и chunked -> 413; + - пиксели: ровно 40 млн -> 201; 50 млн (маленький файл, «бомба») -> 413; + - повреждённые PNG/JPEG (обрезка, CRC, мусор в IDAT), чужие форматы -> 415; + - подмена MIME: PNG как image/jpeg, JPEG как image/png, text/plain, multipart -> 415; + - опасные filename (/, \\, NUL, управляющие, bidi, пустой, 256 символов) -> 400; + - чужой пользователь: upload в чужой чат, metadata, content -> 404; + - message: вложение другого чата / неизвестное -> 404, дубли / 11 ID -> 400, + text="" с вложением -> 201, повтор Idempotency-Key -> то же сообщение; + - оборванная загрузка не ломает реплику; variant без варианта -> 404. + +Запуск: docker compose --profile tools run --rm tester python3 student-tests/attachment_edge_cases.py +или с хоста: python student-tests/attachment_edge_cases.py --base-url http://localhost:8080 +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import http.client +import json +import os +import socket +import struct +import sys +import urllib.parse +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE_PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes() +SAMPLE_JPG = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes() +MIB10 = 10 * 1024 * 1024 + + +# --- синтетические изображения --------------------------------------------- + +def png_chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + + +def png(width: int, height: int, bit_depth: int = 8, color_type: int = 0, idat: bytes | None = None, + extra: bytes = b'') -> bytes: + if idat is None: + row_bytes = (width * bit_depth * {0: 1, 2: 3}[color_type] + 7) // 8 + raw = (b'\x00' + b'\x00' * row_bytes) * height + idat = zlib.compress(raw, 9) + ihdr = struct.pack('>IIBBBBB', width, height, bit_depth, color_type, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', ihdr) + extra + + png_chunk(b'IDAT', idat) + png_chunk(b'IEND', b'')) + + +def png_exactly(size: int) -> bytes: + """Валидный PNG ровно size байт: добивается приватным ancillary-чанком.""" + base = png(8, 8) + pad = size - len(base) - 12 + return png(8, 8, extra=png_chunk(b'zpAD', b'\x00' * pad)) + + +def pixels_png(width: int, height: int) -> bytes: + """1-битный grayscale: 40 млн пикселей занимают ~5 MB raw, ~10 KB zlib.""" + row = b'\x00' + b'\x00' * ((width + 7) // 8) + compressor = zlib.compressobj(9) + idat = b''.join(compressor.compress(row) for _ in range(height)) + compressor.flush() + return png(width, height, bit_depth=1, idat=idat) + + +def corrupt_crc(data: bytes) -> bytes: + i = data.index(b'IDAT') + 6 + return data[:i] + bytes([data[i] ^ 0xFF]) + data[i + 1:] + + +def corrupt_idat(data: bytes) -> bytes: + """Мусор в zlib-потоке с ПРАВИЛЬНЫМ CRC: ловится только распаковкой.""" + return png(320, 200, color_type=2, idat=b'\x78\x9c' + os.urandom(2000)) + + +# --- HTTP ------------------------------------------------------------------ + +class Api: + def __init__(self, base_url: str): + parts = urllib.parse.urlsplit(base_url) + self.host = parts.hostname + self.port = parts.port or 80 + + def conn(self) -> http.client.HTTPConnection: + return http.client.HTTPConnection(self.host, self.port, timeout=60) + + def call(self, method, path, body=None, headers=None, raw=None, chunks=None): + h = dict(headers or {}) + if body is not None: + raw = json.dumps(body).encode() + h['Content-Type'] = 'application/json' + c = self.conn() + try: + if chunks is not None: + c.request(method, '/api/v1' + path, body=chunks, headers=h, encode_chunked=True) + else: + c.request(method, '/api/v1' + path, body=raw, headers=h) + r = c.getresponse() + data = r.read() + except (BrokenPipeError, ConnectionResetError): + # Сервер ответил 413 и закрыл соединение, не дочитав тело. + r = c.getresponse() + data = r.read() + finally: + c.close() + ctype = r.getheader('Content-Type', '') + parsed = json.loads(data) if data and ctype.startswith('application/json') else None + return r.status, parsed, data, r + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + args = p.parse_args() + api = Api(args.base_url) + failed = 0 + + def code(resp): + return resp.get('error', {}).get('code') if isinstance(resp, dict) else None + + def check(label, status, resp, want_status, want_code=None, extra_ok=True): + nonlocal failed + ok = status == want_status and (want_code is None or code(resp) == want_code) and extra_ok + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label}: {status} {code(resp) or ''} " + f"(ожидается {want_status} {want_code or ''})", flush=True) + + def new_user(): + name = 'att' + uuid.uuid4().hex[:12] + password = 'attachment-edge-password' + s, user, _, _ = api.call('POST', '/users', {'username': name, 'display_name': 'Вложения', 'password': password}) + assert s == 201, f'register: {s}' + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + s, session, _, _ = api.call('POST', '/auth/sessions', headers={'Authorization': basic}) + assert s == 201, f'login: {s}' + return user, {'Authorization': 'Bearer ' + session['token']} + + def new_chat(auth, members, title='attachments'): + s, chat, _, _ = api.call('POST', '/chats', {'title': title, 'member_ids': members}, headers=auth) + assert s == 201, f'chat: {s}' + return chat + + def upload(auth, chat_id, data, content_type='image/png', filename='photo.png', **kw): + query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='') + h = dict(auth) + if content_type is not None: + h['Content-Type'] = content_type + return api.call('POST', f'/chats/{chat_id}/attachments{query}', headers=h, raw=data, **kw) + + def message(auth, chat_id, text, ids=None, key=None): + body = {'text': text} + if ids is not None: + body['attachment_ids'] = ids + h = dict(auth, **{'Idempotency-Key': key or uuid.uuid4().hex}) + return api.call('POST', f'/chats/{chat_id}/messages', body, headers=h) + + alice, a_auth = new_user() + bob, b_auth = new_user() + eve, e_auth = new_user() + chat = new_chat(a_auth, [alice['id'], bob['id']]) + other_chat = new_chat(a_auth, [alice['id']], 'other') + eve_chat = new_chat(e_auth, [eve['id']], 'eve') + + # --- успешные границы --------------------------------------------------- + s, att, _, r = upload(a_auth, chat['id'], SAMPLE_JPG, 'image/jpeg', 'фото 😀 v1.0.jpg') + check('JPEG fixture -> 201 ready', s, att, 201, extra_ok=s == 201 and att['status'] == 'ready' + and att['sha256'] == hashlib.sha256(SAMPLE_JPG).hexdigest() and att['size_bytes'] == len(SAMPLE_JPG) + and att['filename'] == 'фото 😀 v1.0.jpg' and att['variants'] == [] and att['error_code'] is None + and 'object_key' not in att) + jpg_id = att['id'] + s, _, data, r = api.call('GET', f'/attachments/{jpg_id}/content', headers=b_auth) + check('участник скачивает JPEG побайтно, без redirect', s, None, 200, + extra_ok=data == SAMPLE_JPG and r.getheader('Content-Type') == 'image/jpeg') + + exact = png_exactly(MIB10) + s, att, _, _ = upload(a_auth, chat['id'], exact) + check('PNG ровно 10 MiB -> 201', s, att, 201, extra_ok=s == 201 and att['size_bytes'] == MIB10) + if s == 201: + s, _, data, _ = api.call('GET', f"/attachments/{att['id']}/content", headers=a_auth) + check('10 MiB скачан побайтно', s, None, 200, extra_ok=hashlib.sha256(data).digest() == hashlib.sha256(exact).digest()) + + s, att, _, _ = upload(a_auth, chat['id'], pixels_png(8000, 5000)) + check('PNG ровно 40 млн пикселей -> 201', s, att, 201) + + # --- размер ------------------------------------------------------------- + s, r, _, _ = upload(a_auth, chat['id'], png_exactly(MIB10 + 1)) + check('10 MiB + 1 байт (Content-Length) -> 413', s, r, 413, 'payload_too_large') + + def big_chunks(): + block = png_exactly(MIB10)[:1024 * 1024] + for _ in range(11): + yield block + + s, r, _, _ = upload(a_auth, chat['id'], None, chunks=big_chunks()) + check('11 MiB chunked без Content-Length -> 413', s, r, 413, 'payload_too_large') + + s, r, _, _ = upload(a_auth, chat['id'], pixels_png(10000, 5000)) + check('PNG-«бомба» 50 млн пикселей (~10 KB файл) -> 413', s, r, 413, 'payload_too_large') + s, r, _, _ = upload(a_auth, chat['id'], png(0x7FFFFFFF, 0x7FFFFFFF, idat=b'x')) + check('PNG 2^31-1 x 2^31-1 в IHDR -> 413', s, r, 413, 'payload_too_large') + + s, r, _, _ = upload(a_auth, chat['id'], b'') + check('пустое тело -> 400', s, r, 400, 'invalid_request') + + # --- повреждённые и чужие форматы -------------------------------------- + for label, data, ctype in [ + ('PNG обрезан пополам', SAMPLE_PNG[:len(SAMPLE_PNG) // 2], 'image/png'), + ('PNG с неверным CRC', corrupt_crc(SAMPLE_PNG), 'image/png'), + ('PNG с мусором в zlib при верном CRC', corrupt_idat(SAMPLE_PNG), 'image/png'), + ('PNG без IEND', SAMPLE_PNG[:-12], 'image/png'), + ('JPEG обрезан', SAMPLE_JPG[:len(SAMPLE_JPG) // 2], 'image/jpeg'), + ('JPEG только сигнатура', b'\xff\xd8\xff\xe0' + b'\x00' * 100, 'image/jpeg'), + ('GIF как image/png', b'GIF89a' + b'\x00' * 100, 'image/png'), + ('случайные байты как image/png', os.urandom(4096), 'image/png'), + ('HTML как image/jpeg', b'', 'image/jpeg'), + ]: + s, r, _, _ = upload(a_auth, chat['id'], data, ctype) + check(label + ' -> 415', s, r, 415, 'unsupported_media_type') + + # --- подмена MIME ------------------------------------------------------- + for label, data, ctype in [ + ('PNG под Content-Type image/jpeg', SAMPLE_PNG, 'image/jpeg'), + ('JPEG под Content-Type image/png', SAMPLE_JPG, 'image/png'), + ('PNG под text/plain', SAMPLE_PNG, 'text/plain'), + ('PNG под image/gif', SAMPLE_PNG, 'image/gif'), + ('multipart/form-data', b'--x\r\n\r\n' + SAMPLE_PNG + b'\r\n--x--', 'multipart/form-data; boundary=x'), + ('без Content-Type', SAMPLE_PNG, None), + ]: + s, r, _, _ = upload(a_auth, chat['id'], data, ctype) + check(label + ' -> 415', s, r, 415, 'unsupported_media_type') + + # --- filename ----------------------------------------------------------- + for label, name in [ + ('../../etc/passwd', '../../etc/passwd'), + ('C:\\boot.png', 'C:\\boot.png'), + ('NUL в имени', 'a\x00.png'), + ('перевод строки', 'a\nb.png'), + ('ESC-последовательность', '\x1b[31mred.png'), + ('bidi override (gnp.exe)', 'photo\u202egnp.exe'), + ('только пробелы', ' '), + ('..', '..'), + ('256 символов', 'a' * 252 + '.png'), + ('пустой filename', ''), + ('нет filename', None), + ]: + s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG, filename=name) + check(f'filename {label} -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG, filename='a' * 251 + '.png') + check('filename ровно 255 символов -> 201', s, r, 201) + + # --- доступ ------------------------------------------------------------- + s, r, _, _ = upload({}, chat['id'], SAMPLE_PNG) + check('upload без токена -> 401', s, r, 401, 'unauthorized') + s, r, _, _ = upload(e_auth, chat['id'], SAMPLE_PNG) + check('посторонний upload в чужой чат -> 404', s, r, 404, 'not_found') + s, r, _, _ = upload(a_auth, str(uuid.uuid4()), SAMPLE_PNG) + check('upload в несуществующий чат -> 404', s, r, 404, 'not_found') + for suffix in ('', '/content'): + s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}{suffix}', headers=e_auth) + check(f'посторонний GET /attachments/id{suffix} -> 404', s, r, 404, 'not_found') + s, r, _, _ = api.call('GET', f'/attachments/{uuid.uuid4()}{suffix}', headers=a_auth) + check(f'неизвестный attachment{suffix} -> 404', s, r, 404, 'not_found') + s, r, _, _ = api.call('GET', '/attachments/not-a-uuid', headers=a_auth) + check('attachment_id не UUID -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}/content?variant=thumbnail', headers=a_auth) + check('ready без варианта thumbnail -> 404', s, r, 404, 'not_found') + s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}/content?variant=../../x', headers=a_auth) + check('variant неизвестного имени -> 400', s, r, 400, 'invalid_request') + + # --- привязка к сообщению ----------------------------------------------- + s, other_att, _, _ = upload(a_auth, other_chat['id'], SAMPLE_PNG) + s, eve_att, _, _ = upload(e_auth, eve_chat['id'], SAMPLE_PNG) + s, r, _, _ = message(a_auth, chat['id'], 'x', [other_att['id']]) + check('вложение другого чата (своего) -> 404', s, r, 404, 'not_found') + s, r, _, _ = message(a_auth, chat['id'], 'x', [eve_att['id']]) + check('вложение чужого чата -> 404', s, r, 404, 'not_found') + s, r, _, _ = message(a_auth, chat['id'], 'x', [jpg_id, str(uuid.uuid4())]) + check('одно из ID неизвестно -> 404, ничего не создано', s, r, 404, 'not_found') + s, r, _, _ = message(a_auth, chat['id'], 'x', [jpg_id, jpg_id]) + check('дубли attachment_ids -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = message(a_auth, chat['id'], 'x', [str(uuid.uuid4()) for _ in range(11)]) + check('11 attachment_ids -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = message(a_auth, chat['id'], 'x', ['not-a-uuid']) + check('attachment_id не UUID -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = message(a_auth, chat['id'], ' ', []) + check('пробельный text без вложений -> 400', s, r, 400, 'invalid_request') + + key = uuid.uuid4().hex + s, m1, _, _ = message(b_auth, chat['id'], '', [jpg_id], key) + check('Bob: text="" + вложение Alice из общего чата -> 201', s, m1, 201, + extra_ok=s == 201 and m1['attachment_ids'] == [jpg_id] and m1['text'] == '') + s, m2, _, _ = message(b_auth, chat['id'], '', [jpg_id], key) + check('повтор Idempotency-Key -> то же сообщение', s, m2, 201, extra_ok=s == 201 and m2['id'] == m1['id']) + s, r, _, _ = message(b_auth, chat['id'], 'другой текст', [], key) + check('тот же ключ, другое тело -> 409', s, r, 409, 'conflict') + s, page, _, _ = api.call('GET', f"/chats/{chat['id']}/messages?limit=100", headers=a_auth) + linked = [m for m in page['items'] if m['attachment_ids']] + check('история: ровно одно сообщение с вложением', s, page, 200, + extra_ok=len(linked) == 1 and linked[0]['attachment_ids'] == [jpg_id]) + + # --- оборванная загрузка ------------------------------------------------ + sock = socket.create_connection((api.host, api.port), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\n" + f"Host: {api.host}\r\nAuthorization: {a_auth['Authorization']}\r\n" + f"Content-Type: image/png\r\nContent-Length: {len(SAMPLE_PNG)}\r\n\r\n") + sock.sendall(head.encode() + SAMPLE_PNG[:len(SAMPLE_PNG) // 3]) + sock.close() + s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG) + check('после оборванной загрузки API работает', s, r, 201) + + # --- формат ошибок ------------------------------------------------------ + s, r, _, resp = upload(a_auth, chat['id'], b'not an image') + check('ошибка в общем формате с request_id', s, r, 415, extra_ok=bool(r) + and r['error']['request_id'] == resp.getheader('X-Request-Id') and bool(resp.getheader('X-Instance-Id'))) + + print('PASS: граничные случаи вложений соответствуют контракту' if not failed + else f'FAIL: {failed} проверок не прошло') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/contract_edge_cases.py b/student-tests/contract_edge_cases.py new file mode 100644 index 00000000..bb63a83b --- /dev/null +++ b/student-tests/contract_edge_cases.py @@ -0,0 +1,80 @@ +"""Граничные случаи контракта: тело при входе и лимит участников чата (stdlib). + +1. POST /auth/sessions с действительным Basic и телом ({bad, {}) -> 400 invalid_request; + без тела -> 201. +2. POST /chats: 101 участник -> 400 invalid_request; ровно 100 -> 201. + +Запуск: docker compose --profile tools run --rm tester python3 student-tests/contract_edge_cases.py +или с хоста: python student-tests/contract_edge_cases.py --base-url http://localhost:8080 +""" +import argparse +import base64 +import json +import os +import sys +import urllib.error +import urllib.request +import uuid + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + base = p.parse_args().base_url.rstrip('/') + '/api/v1' + + def call(method, path, body=None, headers=None, raw=None): + h = dict(headers or {}) + if body is not None: + raw = json.dumps(body).encode() + h['Content-Type'] = 'application/json' + req = urllib.request.Request(base + path, data=raw, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, json.loads(r.read() or b'null') + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b'null') + + def new_user(): + name = 'edge' + uuid.uuid4().hex[:12] + password = 'edge-case-password-1' + status, user = call('POST', '/users', {'username': name, 'display_name': 'Edge case', 'password': password}) + assert status == 201, f'register: {status}' + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + return user, basic + + def code(resp): + return resp.get('error', {}).get('code') if isinstance(resp, dict) else None + + failed = 0 + + def check(label, status, resp, want_status, want_code=None): + nonlocal failed + ok = status == want_status and (want_code is None or code(resp) == want_code) + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label}: {status} {code(resp) or ''} (ожидается {want_status} {want_code or ''})") + + user, basic = new_user() + json_headers = {'Authorization': basic, 'Content-Type': 'application/json'} + + s, r = call('POST', '/auth/sessions', headers=json_headers, raw=b'{bad') + check('вход: Basic + тело {bad', s, r, 400, 'invalid_request') + s, r = call('POST', '/auth/sessions', headers=json_headers, raw=b'{}') + check('вход: Basic + тело {}', s, r, 400, 'invalid_request') + s, r = call('POST', '/auth/sessions', headers=json_headers) + check('вход: Basic без тела (Content-Type: application/json)', s, r, 201) + s, session = call('POST', '/auth/sessions', headers={'Authorization': basic}) + check('вход: Basic без тела', s, session, 201) + + bearer = {'Authorization': 'Bearer ' + session['token']} + others = [new_user()[0]['id'] for _ in range(100)] + s, r = call('POST', '/chats', {'title': '101 members', 'member_ids': [user['id']] + others}, headers=bearer) + check('чат: 101 участник', s, r, 400, 'invalid_request') + s, r = call('POST', '/chats', {'title': '100 members', 'member_ids': [user['id']] + others[:99]}, headers=bearer) + check('чат: ровно 100 участников', s, r, 201) + + print('PASS: тело при входе и лимит участников соответствуют контракту' if not failed else f'FAIL: {failed} проверок не прошло') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 00000000..1bafc900 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/s3_failures.py b/student-tests/s3_failures.py new file mode 100644 index 00000000..c473dcb4 --- /dev/null +++ b/student-tests/s3_failures.py @@ -0,0 +1,185 @@ +"""Приватность bucket вне API и недоступность S3 (stdlib + docker CLI). + +1. Bucket приватен вне API: реальный object_key берётся из PostgreSQL + (docker exec db psql), затем анонимные GET/HEAD/LIST/PUT напрямую в S3 + из сети backend -> 403; S3-порт не опубликован на хосте; LB не + проксирует пути S3. +2. Отказ S3 (docker stop s3): /health/ready 200, сообщения 201, metadata 200, + upload/download -> быстрый 503 в общем формате (не 500, не зависание). + После docker start s3: upload 201, старый файл скачивается побайтно. + +Нужен docker.sock: запускайте только в tester на учебном стенде: + docker compose --profile tools run --rm tester python3 student-tests/s3_failures.py +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/') +S3 = os.environ.get('S3_ENDPOINT', 'http://s3:9000').rstrip('/') +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') +BUCKET = 'messenger-attachments' +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + +failed = 0 + + +def check(label, ok, detail=''): + global failed + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label} {detail}".rstrip(), flush=True) + + +def docker(*args, check_rc=True) -> str: + return subprocess.run(['docker', *args], check=check_rc, capture_output=True, text=True).stdout.strip() + + +def container(service: str) -> str: + ids = docker('ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', f'label=com.docker.compose.service={service}').split() + assert ids, f'no container for {service}' + return ids[0] + + +def http(method, url, data=None, headers=None, timeout=30): + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + started = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read(), time.monotonic() - started + except urllib.error.HTTPError as e: + return e.code, e.read(), time.monotonic() - started + + +def api(method, path, body=None, auth=None, raw=None, ctype=None, extra=None): + h = dict(extra or {}) + if auth: + h['Authorization'] = auth + if body is not None: + raw = json.dumps(body).encode() + ctype = 'application/json' + if ctype: + h['Content-Type'] = ctype + status, data, elapsed = http(method, BASE + '/api/v1' + path, raw, h) + try: + parsed = json.loads(data) if data else None + except ValueError: + parsed = None + return status, parsed, data, elapsed + + +def err(parsed): + return parsed.get('error', {}).get('code') if isinstance(parsed, dict) else None + + +def wait_healthy(cid, seconds=90): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy': + return True + time.sleep(1) + return False + + +def main() -> int: + name = 's3f' + uuid.uuid4().hex[:12] + password = 's3-failure-password' + s, user, _, _ = api('POST', '/users', {'username': name, 'display_name': 'S3', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + s, session, _, _ = api('POST', '/auth/sessions', auth=basic) + auth = 'Bearer ' + session['token'] + s, chat, _, _ = api('POST', '/chats', {'title': 's3', 'member_ids': [user['id']]}, auth=auth) + s, att, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=a.png", auth=auth, raw=SAMPLE, ctype='image/png') + assert s == 201, s + + # --- 1. приватный bucket вне API --------------------------------------- + db = container('db') + key = docker('exec', db, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', + f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + check('object_key есть в PostgreSQL и не совпадает с filename/ID', + bool(key) and 'a.png' not in key and att['id'] not in key) + check('API не раскрывает object_key', key not in json.dumps(att)) + for method, path, data in [ + ('GET', f'/{BUCKET}/{key}', None), + ('HEAD', f'/{BUCKET}/{key}', None), + ('GET', f'/{BUCKET}?list-type=2', None), + ('GET', f'/{BUCKET}/', None), + ('PUT', f'/{BUCKET}/{key}', b'overwrite'), + ('DELETE', f'/{BUCKET}/{key}', None), + ]: + s, _, _ = http(method, S3 + path, data) + check(f'анонимный {method} {path.replace(key, "")} напрямую в S3 -> 403', s == 403, f'({s})') + s, _, data, _ = api('GET', f"/attachments/{att['id']}/content", auth=auth) + check('объект не изменён анонимными PUT/DELETE', hashlib.sha256(data).digest() == hashlib.sha256(SAMPLE).digest()) + s3 = container('s3') + ports = docker('port', s3, check_rc=False) + check('S3 не опубликован на хосте (docker port пуст)', ports == '', repr(ports)) + s, _, _ = http('GET', BASE + f'/{BUCKET}/{key}') + check('LB не проксирует путь S3', s == 404, f'({s})') + host = urllib.parse.urlsplit(S3).hostname + for port, role in [(9333, 'master UI/API'), (8888, 'filer UI/API'), (8080, 'volume'), + (19333, 'master gRPC'), (18888, 'filer gRPC'), (23646, 'admin')]: + try: + socket.create_connection((host, port), timeout=2).close() + reachable = True + except OSError: + reachable = False + check(f'SeaweedFS {role} :{port} недоступен из сети стенда', not reachable) + + # --- 2. отказ S3 ------------------------------------------------------- + docker('stop', '--time', '10', s3) + print('-- s3 остановлен', flush=True) + try: + s, _, _ = http('GET', BASE + '/health/ready') + check('/health/ready при отказе S3 -> 200 (S3 не критичен для чата)', s == 200, f'({s})') + s, m, _, _ = api('POST', f"/chats/{chat['id']}/messages", {'text': 'без S3', 'attachment_ids': [att['id']]}, + auth=auth, extra={'Idempotency-Key': uuid.uuid4().hex}) + check('сообщение со ссылкой на готовое вложение -> 201', s == 201, f'({s})') + s, _, _, _ = api('GET', f"/attachments/{att['id']}", auth=auth) + check('metadata из PostgreSQL -> 200', s == 200, f'({s})') + s, p, _, t = api('POST', f"/chats/{chat['id']}/attachments?filename=b.png", auth=auth, raw=SAMPLE, ctype='image/png') + check('upload -> 503 unavailable', s == 503 and err(p) == 'unavailable', f'({s} {err(p)}, {t:.2f} s)') + check('upload отказал быстрее timeout LB (< 8 s)', t < 8, f'({t:.2f} s)') + s, p, _, t = api('GET', f"/attachments/{att['id']}/content", auth=auth) + check('download -> 503 unavailable', s == 503 and err(p) == 'unavailable', f'({s} {err(p)}, {t:.2f} s)') + check('download отказал быстрее timeout LB (< 8 s)', t < 8, f'({t:.2f} s)') + s, p, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=c.png", auth=auth, raw=b'junk', ctype='image/png') + check('пользовательская ошибка при отказе S3 остаётся 415', s == 415, f'({s})') + intents = docker('exec', db, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', + 'SELECT count(*) FROM upload_intents') + print(f' upload_intents после неудачных upload: {intents}', flush=True) + finally: + docker('start', s3) + print('-- s3 запущен', flush=True) + + check('s3 снова healthy', wait_healthy(s3)) + deadline = time.monotonic() + 30 + while True: + s, new, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=d.png", auth=auth, raw=SAMPLE, ctype='image/png') + if s == 201 or time.monotonic() > deadline: + break + time.sleep(1) + check('после восстановления S3 upload -> 201', s == 201, f'({s})') + s, _, data, _ = api('GET', f"/attachments/{att['id']}/content", auth=auth) + check('старый файл скачивается побайтно', s == 200 and data == SAMPLE, f'({s})') + + print('PASS: bucket приватен вне API, отказ S3 даёт 503 только для вложений' if not failed + else f'FAIL: {failed} проверок не прошло') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/upload_compensation.py b/student-tests/upload_compensation.py new file mode 100644 index 00000000..46bba9de --- /dev/null +++ b/student-tests/upload_compensation.py @@ -0,0 +1,221 @@ +"""Уровень 4: отказ между S3 PUT и COMMIT не оставляет вечных объектов. + + 1. db_error_after_s3_put — процесс API (FAULT_INJECTION, порт 8091 внутри + контейнера реплики) теряет БД после PUT -> 503, фоновая компенсация + сама удаляет объект и intent; + 2. crash_after_s3_put — процесс (порт 8090) умирает после PUT до COMMIT -> + остаются intent и объект; janitor после grace удаляет оба; + 3. свежий intent (моложе grace, «загрузка в процессе») janitor не трогает; + готовое вложение после janitor скачивается побайтно. + + docker compose --profile tools run --rm tester python3 student-tests/upload_compensation.py +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/') +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +GRACE = 20 + +failed = 0 + + +def check(label, ok, detail=''): + global failed + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label} {detail}".rstrip(), flush=True) + + +def docker(*args, stdin=None) -> str: + r = subprocess.run(['docker', *args], capture_output=True, text=True, input=stdin) + if r.returncode: + raise RuntimeError(f'docker {args[0]} failed: {r.stderr.strip()[-300:]}') + return r.stdout.strip() + + +def container(service: str) -> str: + return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', f'label=com.docker.compose.service={service}').split()[0] + + +API = DB = '' + + +def psql(sql: str) -> str: + return docker('exec', DB, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', sql) + + +def intents() -> set[str]: + return set(filter(None, psql('SELECT object_key FROM upload_intents').splitlines())) + + +def s3_exists(*keys: str) -> list: + code = """ +import asyncio, json, sys +from app.storage import storage +async def main(): + await storage.start() + out = [await storage.exists(k) for k in json.load(sys.stdin)] + await storage.close() + print(json.dumps(out)) +asyncio.run(main()) +""" + return json.loads(docker('exec', '-i', API, 'python', '-c', code, stdin=json.dumps(list(keys)))) + + +def run_janitor(grace: int) -> int: + code = f""" +import asyncio +from app.attachments import cleanup_stale_uploads +from app.storage import storage +async def main(): + await storage.start() + print(await cleanup_stale_uploads({grace})) + await storage.close() +asyncio.run(main()) +""" + return int(docker('exec', API, 'python', '-c', code).splitlines()[-1]) + + +def http(method, url, data=None, headers=None, timeout=30): + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +class Client: + def __init__(self, base): + self.base = base + + def call(self, method, path, body=None, auth=None, raw=None, ctype=None): + h = {} + if auth: + h['Authorization'] = auth + if body is not None: + raw, ctype = json.dumps(body).encode(), 'application/json' + if ctype: + h['Content-Type'] = ctype + s, data = http(method, self.base + '/api/v1' + path, raw, h) + try: + return s, json.loads(data) + except ValueError: + return s, data + + def upload(self, auth, chat_id): + return self.call('POST', f'/chats/{chat_id}/attachments?filename=c.png', + auth=auth, raw=SAMPLE, ctype='image/png') + + +def start_faulty(mode: str, port: int) -> Client: + docker('exec', '-d', '-e', f'FAULT_INJECTION={mode}', '-e', f'APP_PORT={port}', + '-e', 'JANITOR_INTERVAL_SECONDS=3600', API, 'python', '-m', 'app.server') + name = docker('inspect', '-f', '{{.Name}}', API).lstrip('/') + base = f'http://{name}:{port}' + for _ in range(60): + try: + if http('GET', base + '/health/ready', timeout=2)[0] == 200: + return Client(base) + except OSError: + pass + time.sleep(0.5) + raise RuntimeError(f'faulty process {mode} did not start') + + +def stop_faulty() -> None: + """Kill the extra app.server processes (not PID 1, the real replica).""" + code = """ +import os, signal +for p in filter(str.isdigit, os.listdir('/proc')): + if int(p) in (1, os.getpid()): + continue + try: + cmd = open(f'/proc/{p}/cmdline', 'rb').read() + except OSError: + continue + if b'app.server' in cmd: + os.kill(int(p), signal.SIGKILL) +""" + docker('exec', API, 'python', '-c', code) + + +def main() -> int: + global API, DB + API, DB = container('api'), container('db') + lb = Client(BASE) + + name = 'comp' + uuid.uuid4().hex[:12] + password = 'compensation-password-1' + lb.call('POST', '/users', {'username': name, 'display_name': 'Comp', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + auth = 'Bearer ' + lb.call('POST', '/auth/sessions', auth=basic)[1]['token'] + me = lb.call('GET', '/users/me', auth=auth)[1] + chat = lb.call('POST', '/chats', {'title': 'compensation', 'member_ids': [me['id']]}, auth=auth)[1] + ready = lb.upload(auth, chat['id'])[1] + + try: + # 1. DB lost after PUT -> 503 + background compensation + before = intents() + s, _ = start_faulty('db_error_after_s3_put', 8091).upload(auth, chat['id']) + time.sleep(2) + check('db_error_after_s3_put: 503', s == 503, f'({s})') + check('db_error_after_s3_put: компенсация убрала intent', intents() - before == set()) + + # 2. crash after PUT -> intent + object stay + before = intents() + try: + s, _ = start_faulty('crash_after_s3_put', 8090).upload(auth, chat['id']) + result = f'HTTP {s}' + except OSError as exc: + result = type(exc).__name__ + crash_keys = sorted(intents() - before) + check('crash_after_s3_put: клиент не получил 201', '201' not in result, f'({result})') + check('crash_after_s3_put: остались intent и объект', + len(crash_keys) == 1 and s3_exists(*crash_keys) == [True]) + finally: + stop_faulty() + + check('janitor до grace ничего не удаляет', + run_janitor(GRACE) == 0 and set(crash_keys) <= intents()) + + print(f'-- ждём grace {GRACE + 1} s', flush=True) + time.sleep(GRACE + 1) + + # 3. fresh in-flight upload (younger than grace) must survive + inflight = f'originals/{uuid.uuid4().hex}' + psql(f"INSERT INTO upload_intents (object_key, attachment_id, chat_id) " + f"VALUES ('{inflight}', '{uuid.uuid4()}', '{chat['id']}')") + + cleaned = run_janitor(GRACE) + left = intents() + check('janitor после grace удалил intent от crash', cleaned >= 1 and not set(crash_keys) & left, + f'(cleaned={cleaned})') + check('janitor удалил объект от crash', s3_exists(*crash_keys) == [False]) + check('свежий intent не тронут', inflight in left) + psql(f"DELETE FROM upload_intents WHERE object_key = '{inflight}'") + + s, body = lb.call('GET', f"/attachments/{ready['id']}/content", auth=auth) + check('готовое вложение цело', s == 200 and hashlib.sha256(body).hexdigest() == ready['sha256']) + + print('PASS: компенсация и janitor не оставляют вечных объектов' if not failed + else f'FAIL: {failed} проверок не прошли') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/upload_failover.py b/student-tests/upload_failover.py new file mode 100644 index 00000000..63e6a959 --- /dev/null +++ b/student-tests/upload_failover.py @@ -0,0 +1,105 @@ +"""Upload -> остановка обслужившей реплики -> download через другую, для КАЖДОЙ реплики. + +Публичный failover-тест останавливает ту реплику, что ему ответила (при +round-robin — обычно одну и ту же). Здесь цель выбирается явно: файл +загружается, пока X-Instance-Id не совпадёт с целевой репликой, затем она +останавливается (docker stop, остаётся остановленной на время проверки), +файл скачивается через LB и сравнивается по SHA-256; реплика запускается снова. + + docker compose --profile tools run --rm tester python3 student-tests/upload_failover.py +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/') +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def docker(*args) -> str: + return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip() + + +def call(method, path, body=None, auth=None, raw=None, ctype=None): + h = {} + if auth: + h['Authorization'] = auth + if body is not None: + raw, ctype = json.dumps(body).encode(), 'application/json' + if ctype: + h['Content-Type'] = ctype + req = urllib.request.Request(BASE + '/api/v1' + path, data=raw, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read(), r.headers.get('X-Instance-Id') + except urllib.error.HTTPError as e: + return e.code, e.read(), e.headers.get('X-Instance-Id') + + +def main() -> int: + failed = 0 + replicas = docker('ps', '-q', '--no-trunc', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', 'label=com.docker.compose.service=api').split() + assert len(replicas) >= 2, 'need >= 2 running api replicas' + + name = 'ufo' + uuid.uuid4().hex[:12] + password = 'upload-failover-password' + call('POST', '/users', {'username': name, 'display_name': 'UFO', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + auth = 'Bearer ' + json.loads(call('POST', '/auth/sessions', auth=basic)[1])['token'] + me = json.loads(call('GET', '/users/me', auth=auth)[1]) + chat = json.loads(call('POST', '/chats', {'title': 'ufo', 'member_ids': [me['id']]}, auth=auth)[1]) + digest = hashlib.sha256(SAMPLE).hexdigest() + + for target in replicas: + alias = target[:12] + for _ in range(20): + s, body, instance = call('POST', f"/chats/{chat['id']}/attachments?filename=ufo.png", + auth=auth, raw=SAMPLE, ctype='image/png') + if s == 201 and target.startswith(instance): + break + else: + print(f'FAIL could not hit replica {alias}') + failed += 1 + continue + att = json.loads(body) + docker('stop', '--time', '15', target) + time.sleep(1) + try: + served = set() + ok = True + for _ in range(4): + s, data, instance = call('GET', f"/attachments/{att['id']}/content", auth=auth) + served.add(instance) + ok &= s == 200 and hashlib.sha256(data).hexdigest() == digest + ok &= alias not in served + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} upload через {alias} -> {alias} остановлена -> " + f"download через {sorted(served)}: SHA-256 {'совпал' if ok else 'НЕ совпал'}", flush=True) + finally: + docker('start', target) + for _ in range(60): + if docker('inspect', '-f', '{{.State.Health.Status}}', target) == 'healthy': + break + time.sleep(1) + time.sleep(3) + + print('PASS: файл доступен после остановки любой обслужившей реплики' if not failed + else f'FAIL: {failed}') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..bb99a14d --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,230 @@ +"""Эксперимент: пик памяти API при N одновременных загрузках (stdlib + docker CLI). + +Гипотеза: память реплики ограничена конструкцией, а не числом/размером +загрузок: тело пишется в SpooledTemporaryFile (в RAM <= UPLOAD_SPOOL_MEMORY_BYTES +на загрузку), PNG проверяется потоково, одновременно <= MAX_CONCURRENT_UPLOADS +на реплику. Ожидаем прирост пика RSS << N x размер файла. + +Методика: перезапуск API-реплик (сброс high-water mark) -> базовые VmRSS/VmHWM -> +N потоков одновременно (Barrier) загружают валидный PNG из шума ~9.5 MiB -> +во время загрузок внутри каждой реплики каждые 100 мс читается VmRSS процесса +API (/proc/1/status) и anon cgroup (без page cache временных файлов) -> пик +минус базовое значение -> проверка SHA-256. VmHWM не годится: стартовый пик +импорта библиотек (~170 MiB) выше пика загрузок. + + docker compose --profile tools run --rm tester python3 student-tests/upload_memory.py --concurrency 12 +""" +from __future__ import annotations + +import argparse +import base64 +import collections +import hashlib +import json +import os +import struct +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +import zlib + +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') + + +def docker(*args) -> str: + return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip() + + +def service_containers(service: str) -> list[str]: + return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', f'label=com.docker.compose.service={service}').split() + + +def api_containers() -> list[str]: + return service_containers('api') + + +def memory(cid: str) -> dict: + out = docker('exec', cid, 'sh', '-c', + 'grep -E "^Vm(RSS|HWM)" /proc/1/status; ' + 'echo "peak $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0)"; ' + 'echo "current $(cat /sys/fs/cgroup/memory.current)"; ' + 'grep -E "^anon " /sys/fs/cgroup/memory.stat') + values = {} + for line in out.splitlines(): + name, value = line.replace(':', '').split()[:2] + values[name] = int(value) * (1024 if name.startswith('Vm') else 1) + return {k: round(v / 2 ** 20, 1) for k, v in values.items()} + + +SAMPLER = ('while :; do r=$(grep VmRSS /proc/1/status | tr -s " " | cut -d" " -f2); ' + 'a=$(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2); ' + 'echo "$r $a"; sleep 0.1; done') + + +class Sampler: + def __init__(self, cid: str): + self.proc = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', SAMPLER], + stdout=subprocess.PIPE, text=True) + self.rss = self.anon = 0 + self.samples = 0 + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + + def _read(self): + for line in self.proc.stdout: + try: + rss_kib, anon = map(int, line.split()) + except ValueError: + continue + self.rss = max(self.rss, rss_kib * 1024) + self.anon = max(self.anon, anon) + self.samples += 1 + + def stop(self) -> dict: + self.proc.terminate() + self.proc.wait() + return {'VmRSS_peak': round(self.rss / 2 ** 20, 1), 'anon_peak': round(self.anon / 2 ** 20, 1), + 'samples': self.samples} + + +def noise_png(target_bytes: int) -> bytes: + width = 1800 + height = target_bytes // (width * 3 + 1) + raw = b''.join(b'\x00' + os.urandom(width * 3) for _ in range(height)) + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'') + + +def call(base, method, path, body=None, auth=None, raw=None, ctype=None, timeout=120): + h = {} + if auth: + h['Authorization'] = auth + if body is not None: + raw, ctype = json.dumps(body).encode(), 'application/json' + if ctype: + h['Content-Type'] = ctype + req = urllib.request.Request(base + '/api/v1' + path, data=raw, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read(), r.headers.get('X-Instance-Id') + except urllib.error.HTTPError as e: + return e.code, e.read(), e.headers.get('X-Instance-Id') + + +def wait_ready(base, seconds=90): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base + '/health/ready', timeout=2) as r: + if r.status == 200: + return + except Exception: + pass + time.sleep(1) + raise SystemExit('API is not ready') + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://lb:8080')) + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--size-mib', type=float, default=9.5) + p.add_argument('--no-restart', action='store_true') + p.add_argument('--output') + args = p.parse_args() + base = args.base_url.rstrip('/') + + data = noise_png(int(args.size_mib * 2 ** 20)) + digest = hashlib.sha256(data).hexdigest() + print(f'file: {len(data)} bytes ({len(data) / 2 ** 20:.2f} MiB), concurrency {args.concurrency}', flush=True) + + containers = api_containers() + if not args.no_restart: + docker('restart', '--time', '15', *containers) + time.sleep(3) + wait_ready(base) + time.sleep(3) + + name = 'mem' + uuid.uuid4().hex[:12] + password = 'upload-memory-password' + call(base, 'POST', '/users', {'username': name, 'display_name': 'Mem', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + s, session, _ = call(base, 'POST', '/auth/sessions', auth=basic) + auth = 'Bearer ' + json.loads(session)['token'] + user = json.loads(call(base, 'GET', '/users/me', auth=auth)[1]) + chat = json.loads(call(base, 'POST', '/chats', {'title': 'mem', 'member_ids': [user['id']]}, auth=auth)[1]) + + before = {c[:12]: memory(c) for c in containers} + samplers = {c[:12]: Sampler(c) for c in containers} + s3 = service_containers('s3')[0] + s3_before = memory(s3) + s3_sampler = Sampler(s3) + time.sleep(1) + barrier = threading.Barrier(args.concurrency) + results = [] + lock = threading.Lock() + + def worker(): + barrier.wait() + t = time.monotonic() + s, body, instance = call(base, 'POST', f"/chats/{chat['id']}/attachments?filename=noise.png", + auth=auth, raw=data, ctype='image/png') + with lock: + results.append((s, time.monotonic() - t, instance, body)) + + started = time.monotonic() + threads = [threading.Thread(target=worker) for _ in range(args.concurrency)] + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - started + time.sleep(0.5) + peaks = {c: sm.stop() for c, sm in samplers.items()} + s3_peak = s3_sampler.stop() + after = {c[:12]: memory(c) for c in containers} + + ok = [r for r in results if r[0] == 201] + sha_ok = all(json.loads(r[3])['sha256'] == digest for r in ok) + if ok: + att = json.loads(ok[0][3]) + s, content, _ = call(base, 'GET', f"/attachments/{att['id']}/content", auth=auth) + sha_ok = sha_ok and hashlib.sha256(content).hexdigest() == digest + latencies = sorted(r[1] for r in ok) + summary = { + 'concurrency': args.concurrency, + 'file_bytes': len(data), + 'statuses': dict(collections.Counter(r[0] for r in results)), + 'per_instance': dict(collections.Counter(r[2] for r in results)), + 'wall_seconds': round(wall, 2), + 'latency_p50_s': round(latencies[len(latencies) // 2], 2) if latencies else None, + 'latency_max_s': round(latencies[-1], 2) if latencies else None, + 'sha256_ok': sha_ok, + 'memory_mib_before': before, + 'memory_mib_peak_during': peaks, + 'memory_mib_after': after, + 'rss_peak_increase_mib': { + c: round(peaks[c]['VmRSS_peak'] - before[c]['VmRSS'], 1) for c in peaks + }, + 'anon_peak_increase_mib': { + c: round(peaks[c]['anon_peak'] - before[c]['anon'], 1) for c in peaks + }, + 's3_server_mib': {'VmRSS_before': s3_before['VmRSS'], **s3_peak}, + 'naive_in_memory_estimate_mib': round(args.concurrency * len(data) / 2 ** 20, 1), + } + text = json.dumps(summary, ensure_ascii=False, indent=2) + print(text) + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(text + '\n') + return 0 if ok and sha_ok and all(r[0] in (201, 429) for r in results) else 1 + + +if __name__ == '__main__': + raise SystemExit(main())