From fe4a32bb79301acfc52cc47ad6b717dd69066830 Mon Sep 17 00:00:00 2001 From: masha Date: Fri, 30 Oct 2026 16:53:49 +0300 Subject: [PATCH] Done Track 4 --- .env.example | 20 -- .gitignore | 1 + Dockerfile | 4 +- Dockerfile.example | 8 - REPORT.md | 136 +++++++- app/attachments.py | 441 +++++++++++++++++++++++++ app/config.py | 68 ++++ app/errors.py | 23 ++ app/images.py | 264 +++++++++++++++ app/main.py | 108 +++++- app/models.py | 56 +++- app/repositories.py | 283 +++++++++++++++- app/services.py | 37 ++- app/storage.py | 172 ++++++++++ compose.example.yaml | 12 - compose.yaml | 112 ++++++- evidence/api-no-local-files.txt | 14 + evidence/attachment-edge-cases.txt | 63 ++++ evidence/bootstrap-idempotent.txt | 30 ++ evidence/failover.txt | 24 ++ evidence/persistence.txt | 22 ++ evidence/s3-failures.txt | 34 ++ evidence/smoke.txt | 1 + evidence/upload-compensation.txt | 11 + evidence/upload-failover.txt | 3 + evidence/upload-memory-c12.json | 74 +++++ evidence/upload-memory-c20.json | 74 +++++ infra/haproxy/haproxy.cfg | 6 + infra/seaweedfs/s3.json.template | 22 ++ infra/seaweedfs/start.sh | 24 ++ migrations/004_attachments.sql | 68 ++++ requirements.txt | 7 +- scripts/init-secrets.sh | 7 + scripts/lib.sh | 2 +- scripts/restart.sh | 21 +- scripts/s3_bootstrap.py | 161 +++++++++ student-tests/attachment_edge_cases.py | 327 ++++++++++++++++++ student-tests/fixtures/sample.jpg | Bin 0 -> 4236 bytes student-tests/s3_failures.py | 185 +++++++++++ student-tests/upload_compensation.py | 221 +++++++++++++ student-tests/upload_failover.py | 105 ++++++ student-tests/upload_memory.py | 230 +++++++++++++ 42 files changed, 3363 insertions(+), 118 deletions(-) delete mode 100644 .env.example delete mode 100644 Dockerfile.example create mode 100644 app/attachments.py create mode 100644 app/images.py create mode 100644 app/storage.py delete mode 100644 compose.example.yaml create mode 100644 evidence/api-no-local-files.txt create mode 100644 evidence/attachment-edge-cases.txt create mode 100644 evidence/bootstrap-idempotent.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/s3-failures.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/upload-compensation.txt create mode 100644 evidence/upload-failover.txt create mode 100644 evidence/upload-memory-c12.json create mode 100644 evidence/upload-memory-c20.json create mode 100644 infra/seaweedfs/s3.json.template create mode 100644 infra/seaweedfs/start.sh create mode 100644 migrations/004_attachments.sql create mode 100644 scripts/s3_bootstrap.py create mode 100644 student-tests/attachment_edge_cases.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/s3_failures.py create mode 100644 student-tests/upload_compensation.py create mode 100644 student-tests/upload_failover.py create mode 100644 student-tests/upload_memory.py diff --git a/.env.example b/.env.example deleted file mode 100644 index d99d26a2..00000000 --- a/.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 diff --git a/.gitignore b/.gitignore index f0f39acc..e216ea04 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ coverage/ # Inspect git diff --cached: ignore rules are not a secret scanner. # Local secrets (scripts/init-secrets.sh) infra/secrets/*.txt +ЗАЩИТА.md \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 17b9582a..ef56e3f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Messenger API (lab3). One image for the "api" and "migrate" services. +# Messenger API (lab4). One image for the "api", "migrate" and "s3-init" services. FROM python:3.12.7-slim WORKDIR /app @@ -10,7 +10,7 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY app ./app -COPY scripts/migrate.py ./scripts/migrate.py +COPY scripts/migrate.py scripts/s3_bootstrap.py ./scripts/ COPY migrations ./migrations RUN useradd --create-home --uid 10001 appuser diff --git a/Dockerfile.example b/Dockerfile.example deleted file mode 100644 index e333a12e..00000000 --- a/Dockerfile.example +++ /dev/null @@ -1,8 +0,0 @@ -# Teaching outline, not a runnable Dockerfile. Implement your own Dockerfile. -# 1. Choose a runtime/toolchain version and pin the base image tag/digest. -# 2. Copy dependency manifests first; install from the language lockfile. -# 3. Copy source and build. For compiled languages use a separate runtime stage. -# 4. Run the HTTP process as PID 1; listen on 0.0.0.0:8080. -# 5. Add non-root, signals and healthcheck according to the current rubric. -# 6. Pass runtime config/secrets at startup, never COPY .env or secret files. -# This file deliberately has no FROM/CMD: selecting/implementing them is the task. diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..77873c1c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,137 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Максимова, БВТ2303 (TODO: полное ФИО) +- Целевая оценка; выбранный трек 5: 4 (трек не выбирается) +- Commit/PR: TODO после публикации (ветка `solution/maksimova_bvt2303`) +- Использованные источники, библиотеки и помощь ИИ: FastAPI, SQLAlchemy/asyncpg, argon2-cffi, redis-py, aiobotocore/botocore, Pillow (только для JPEG); SeaweedFS, PostgreSQL, Valkey, HAProxy. Документация: AWS S3 PutObject/SigV4, SeaweedFS wiki (S3 API, `s3.json` identities), спецификация PNG (RFC 2083), HAProxy 3.0 configuration manual. Помощь ИИ (Claude Code): ревью и доработка кода вложений, написание тестов в `student-tests/`, проведение экспериментов и черновик этого отчёта; все результаты получены реальными прогонами на стенде и проверены. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверено на: Windows 11 + Docker Desktop 29.3.0 (WSL2, kernel 6.6.87, 14 vCPU, 7.4 GiB RAM), Docker Compose 5.1.0, Python 3.13 на хосте (для `make test`; достаточно 3.9+). + +| Роль т | Service т | Образ | Health / readiness | Volume | Сеть / порт | Credentials | +|---------------------|-----------|--------------------------------------------|-----------------------------------------------------------|-----------------------------------------|------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------| +| LB | `lb` | `haproxy:3.0.11-alpine` | `/lb-health`; backend-check `GET /health/ready` раз в 1 s | — | edge; `${LB_PORT:-8080}` на хосте, stats только `127.0.0.1:8404` | — | +| API × 2 | `api` | сборка `Dockerfile` (`python:3.12.7-slim`) | `/health/ready` (PostgreSQL + не в drain) | нет | edge + backend, порт не публикуется | Docker secrets: `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` | +| PostgreSQL | `db` | `postgres:16.4-alpine` | `pg_isready` | `postgres_data` | backend (internal) | `postgres_password` | +| Valkey (кэш сессий) | `valkey` | `valkey/valkey:8.0.1-alpine` | `PING` | нет (кэш, источник истины — PostgreSQL) | backend | `valkey_password` | +| S3 | `s3` | `chrislusf/seaweedfs:3.97` | `GET :9000/healthz` | `s3_data` | backend, порт не публикуется; master/filer/volume слушают только `127.0.0.1` внутри контейнера | admin + API ключи из secrets подставляются в `s3.json` при старте | +| Bootstrap bucket | `s3-init` | образ API | one-shot, `service_completed_successfully` | — | backend | единственный, кто получает admin-ключ S3 | +| Миграции | `migrate` | образ API | one-shot | — | backend | `database_url` | + +```sh +sh scripts/init-secrets.sh # создаёт infra/secrets/*.txt (случайные значения, не перезаписывает, не печатает) +cp .env.example .env # при занятом 8080 задайте LB_PORT=8081 +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# порядок: db, valkey, s3 -> healthy -> migrate (SQL migrations/*.sql, повтор = SKIP) +# s3-init (bucket + самопроверки приватности/прав, повтор = keep) -> api x2 healthy -> lb +curl -fsS http://localhost:${LB_PORT:-8080}/health/ready +make test BASE_URL=http://localhost:${LB_PORT:-8080} # на Windows: PYTHONUTF8=1 python tests/smoke.py --base-url ... + +# hook-тесты и собственные проверки — в Linux-контейнере tester (docker CLI + docker.sock): +docker compose --profile tools run --rm tester make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://lb:8080 +docker compose --profile tools run --rm tester make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://lb:8080 +docker compose --profile tools run --rm tester sh scripts/start-api.sh # вернуть остановленную реплику +docker compose --profile tools run --rm tester python3 student-tests/.py + +docker compose stop # безопасная остановка, данные сохраняются +docker compose down -v # НАМЕРЕННОЕ удаление всех учебных данных (volumes) +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client -> HAProxy (lb) -> api x2 --+--> PostgreSQL: users, chats, messages, sessions, idempotency_keys, + | attachments (метаданные), message_attachments, upload_intents + +--> Valkey: кэш сессий (tombstone при logout) + +--> SeaweedFS S3: приватный bucket messenger-attachments/originals/ +``` + +Перенесено из lab3 без изменения контракта: регистрация/Basic→Bearer, сессии в PostgreSQL с кэшем в Valkey, идемпотентная отправка сообщений, `X-Instance-Id`, graceful drain, HAProxy с active check, hooks `restart.sh`/`stop-one.sh`. Изменено: `CreateMessage` принимает `attachment_ids` (по умолчанию `[]`) и `text=""` при наличии вложений; fingerprint идемпотентности без вложений совпадает с lab3, так что старые ключи повторяются корректно. + +Модель. `attachments` (id, chat_id, owner_id, filename — только для отображения, content_type, size, sha256, width/height, status=`ready`, `object_key UNIQUE`) привязано к чату при загрузке. `message_attachments(message_id, attachment_id, position)` хранит порядок; одно вложение можно сослать из нескольких сообщений чата. Миграция `004_attachments.sql` — expand-only. + +Upload (`app/attachments.py`), всё до сохранения байтов: +1. filename (400: `/`, `\`, NUL, Cc/Cf/bidi, пустое/только точки, >255) → членство в чате (404) → `Content-Type` ровно `image/png|image/jpeg` (415, multipart не принимается) → `Content-Length > 10 MiB` → 413 **без чтения тела**; +2. слот загрузки: ≤ `MAX_CONCURRENT_UPLOADS=8` на реплику, ожидание ≤10 s, иначе 429; +3. тело читается потоково в `SpooledTemporaryFile` (в RAM ≤ 1 MiB на загрузку), считается размер и SHA-256 на лету; превышение 10 MiB в chunked → 413; приём дольше 60 s → 400; +4. проверка содержимого в потоке: формат по magic bytes и совпадение с заявленным MIME; PNG — собственный потоковый валидатор (CRC каждого чанка, IHDR, распаковка IDAT кусками по 64 KiB с подсчётом и сравнением с ожидаемым размером, без хранения пикселей); лимит 40 млн пикселей проверяется по IHDR/SOF **до** распаковки (413); JPEG — Pillow `draft()` 1/8 + `load()` (повреждённый поток → 415); +5. `INSERT upload_intent` + COMMIT → S3 PUT (SigV4 подписывает SHA-256 тела) → одна транзакция «DELETE intent + INSERT attachment» → **201 ready**. Честный 201 возможен только после этого COMMIT: и байты, и метаданные уже устойчивы. + +Download: авторизация по `chat_members` в PostgreSQL (чужое/неизвестное → 404), затем потоковая отдача из S3 через API, без redirect, с истинным `Content-Type`, `Content-Length`, `nosniff`, `Cache-Control: private, no-store`. Приватный bucket не заменяет ACL: случайный ключ можно утечь через лог, а права участника чата знает только приложение. + +Согласованность БД/S3 (ответ на «PUT прошёл, а COMMIT нет»): + +| Отказ | Что делает система | +|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| PUT упал / истёк дедлайн | 503 сразу. Исход PUT неизвестен (S3 может дописать объект **после** нашего DELETE), поэтому объект удаляется best effort в фоне, а intent **остаётся**; janitor удалит объект ещё раз после grace и только потом intent | +| COMMIT упал | 503 сразу. Исход COMMIT тоже неизвестен (мог пройти, а потерялся ответ). Фоновая компенсация берёт `SELECT … FOR UPDATE` intent: intent нет → COMMIT прошёл, объект не трогаем; иначе DELETE объекта → DELETE intent | +| Процесс умер после PUT | intent остался; janitor каждой реплики (`JANITOR_INTERVAL_SECONDS=60`, jitter) через `UPLOAD_GRACE_SECONDS=900` удаляет объект и intent; `FOR UPDATE SKIP LOCKED` — реплики не мешают друг другу и финализирующейся загрузке | +| Строка без объекта (потеря байтов вне приложения) | download → 500 `internal_error` и ERROR в логе; метаданные не удаляются (сообщения на них ссылаются) | + +Инвариант: объект без строки существует, только пока жив его intent; объект закоммиченной строки никогда не удаляется. Grace 900 s ≫ максимального времени upload (60 s приём + 16 s PUT). + +Недоступность S3. `/health/ready` S3 не проверяет: без S3 деградируют только вложения (503), чаты и сообщения работают; иначе отказ S3 вывел бы из LB все реплики. Каждый S3-вызов ограничен общим дедлайном `S3_OPERATION_TIMEOUT_SECONDS=6` (DNS + повторы) — меньше `timeout server 10s` у LB; для PUT дедлайн `6 s + size / 1 MiB/s` (≤16 s для 10 MiB), а HAProxy даёт POST вложения 40 s. L7-повторы HAProxy для `/api/v1/attachments/` отключены: 503 там означает отказ общего S3, повтор на другой реплике бесполезен. + +Права S3 (`infra/seaweedfs/s3.json.template`): identity `admin` есть только у `s3-init`; identity `messenger-api` — `Read/Write/List` только на `messenger-attachments` (Write нужен для DELETE janitor-а). Анонимного доступа нет. Admin UI/API SeaweedFS (master 9333, filer 8888, volume 8080, gRPC) слушают только `127.0.0.1` внутри контейнера. + +*Соответствие `.env.example`:** `APP_HOST/APP_PORT`, `SESSION_TTL_SECONDS`, `DATABASE_URL_FILE`, `SESSION_STORE=valkey`, `CACHE_URL_FILE`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS`, `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`, `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` читаются `app/config.py` под теми же именами. Дополнительно в `compose.yaml` (`x-api-env`): `S3_OBJECT_PREFIX`, `S3_CONNECT/READ/OPERATION_TIMEOUT_SECONDS`, `S3_PUT_MIN_BYTES_PER_SECOND`, `UPLOAD_SPOOL_MEMORY_BYTES`, `UPLOAD_TIMEOUT_SECONDS`, `MAX_CONCURRENT_UPLOADS`, `UPLOAD_SLOT_WAIT_SECONDS`, `UPLOAD_GRACE_SECONDS`, `JANITOR_INTERVAL_SECONDS`, `FAULT_INJECTION` (только для экспериментов, пусто в норме). `APP_ENV` не используется. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Все файлы — в `evidence/`, тесты — в `student-tests/`; прогоны 2026-10-30 на стенде выше. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +|-----------------------------------------------------------------------------------------------|-------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------| +| Публичный smoke | `make test BASE_URL=http://localhost:8081` | `PASS: lab4 public API scenario` (SHA-256, 404 постороннему) | `smoke.txt` | +| 3: S3 в стенде, метаданные в PG, байты в приватном bucket | `student-tests/s3_failures.py`; `api-no-local-files.txt` | object_key только в PG; анонимные GET/HEAD/LIST/PUT/DELETE напрямую в S3 → 403; у API нет volumes, `/tmp` пуст | `s3-failures.txt`, `api-no-local-files.txt` | +| 3: PNG/JPEG ≤10 MiB, ≤40 Мпикс, MIME по содержимому, опасные filename | `student-tests/attachment_edge_cases.py` | 60/60 OK: ровно 10 MiB и ровно 40 Мпикс → 201; +1 байт → 413; бомба 50 Мпикс (~10 KB) → 413 | `attachment-edge-cases.txt` | +| 3: upload → стоп обслужившей реплики → download через другую | `make test-failover` ×2; `student-tests/upload_failover.py` | failover: 0 ошибок, восстановление записи 4.0/4.2 s; для **каждой** из двух реплик SHA-256 совпал после её остановки | `failover.txt`, `upload-failover.txt` | +| 3: restart с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; те же ID контейнеров db/s3/valkey/api/lb с новым `StartedAt`, volumes на месте | `persistence.txt` | +| 4: тесты большого файла, повреждения, подмены MIME, чужого id, вложения другого чата; нет 500 | `attachment_edge_cases.py` | обрезка/CRC/мусор в zlib при верном CRC/без IEND → 415; PNG как JPEG, JPEG как PNG, text/plain, multipart → 415; вложение чужого и своего другого чата → 404; ни одного 500 | `attachment-edge-cases.txt` | +| 4: компенсация/сверка/очистка с grace | `student-tests/upload_compensation.py` | DB error после PUT → 503 + фоновая компенсация, intent не остался; crash после PUT → intent + объект, janitor до grace их не трогает, после grace удаляет оба; свежий intent и готовое вложение целы | `upload-compensation.txt` | +| 4: лимиты RAM, ≥10 одновременных загрузок | `student-tests/upload_memory.py --concurrency 12/20` | 12 × 9.5 MiB: прирост пика RSS 13–14 MiB на реплику (наивно 57 MiB данных на реплику) | `upload-memory-c12.json`, `upload-memory-c20.json` | +| 4: идемпотентный bootstrap, отдельные ограниченные credentials | `docker compose run --rm s3-init` ×2, `migrate` | `keep bucket`, все самопроверки PASS: API-ключ не создаёт/не удаляет bucket, не видит чужих; миграции SKIP | `bootstrap-idempotent.txt` | +| 4: админ-консоль закрыта, private bucket проверен вне API | `s3_failures.py` из сети стенда | порты 9333/8888/8080/19333/18888/23646 недоступны; S3 не опубликован на хосте; анонимные запросы → 403 | `s3-failures.txt` | +| 3/4: недоступность S3 | `s3_failures.py` (docker stop s3) | ready 200, сообщения 201, metadata 200; upload/download → 503 за 6.0–6.2 s; 415 остаётся 415; после старта S3 всё работает, старый файл цел | `s3-failures.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Память при одновременных загрузках (уровень 4) -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Гипотеза: пик памяти API определяется конструкцией (spool ≤1 MiB/загрузку, потоковая проверка PNG, ≤8 слотов), а не числом × размером файлов. +Методика: `upload_memory.py` перезапускает обе реплики (сброс состояния), N потоков стартуют одновременно (Barrier) и загружают валидный PNG из шума 9 962 537 байт (9.5 MiB, несжимаемый — худший случай для проверки IDAT); внутри каждой реплики и S3 каждые 100 мс читаются `VmRSS` процесса и `anon` cgroup. `VmHWM` не подходит: пик импорта библиотек при старте (~170 MiB) выше пика загрузок. Лимиты: API 0.5 CPU / 256 MiB, S3 1 CPU / 1 GiB. + +| N (на реплику) | Статусы | Wall, s | p50 / max, s | Пик RSS API − базовый (≈105 MiB) | Наивная оценка «всё в RAM» на реплику | Пик RSS SeaweedFS | +|------------------------|-----------|-----------|----------------|------------------------------------|-----------------------------------------|---------------------| +| 12 (6+6) | 12 × 201 | 5.2 | 5.0 / 5.2 | +12.9 / +13.7 MiB | 57 MiB | 546 MiB | +| 20 (10+10, > 8 слотов) | 20 × 201 | 9.9 | 8.6 / 9.9 | +16.5 / +17.7 MiB | 95 MiB | 779 MiB | + +Гипотеза для API подтверждена: прирост ~2 MiB на одновременную загрузку и почти не растёт сверх 8 слотов (лишние ждут слот; в одном из повторов при N=20 двое получили 429 после 10 s ожидания — ожидаемый backpressure). SHA-256 проверен. + +Побочная находка: узким местом оказался S3-сервер. С исходным лимитом 512 MiB SeaweedFS при 12 параллельных PUT дошёл до 519 MiB и был убит (контейнер перезапущен, все загрузки — 503). Исправлено: лимит 1 GiB и `GOMEMLIMIT=600MiB` (мягкий лимит Go GC); после нагрузки RSS возвращается к ~210 MiB. Это одно измерение на одном ноутбуке, а не ёмкость SeaweedFS вообще. + +### Компенсация отказа между S3 и БД (уровень 4) + +Найденный дефект (до исправления). Сверка содержимого bucket со строками БД на «грязном» стенде нашла 12 объектов без строки и без intent — то, что запрещено инвариантом. Причина: при истёкшем дедлайне PUT компенсация удаляла объект **и intent**, а SeaweedFS дописывал PUT после DELETE. Второй риск того же рода: при неоднозначном COMMIT немедленное удаление объекта оставило бы строку без байтов. Исправлено (см. таблицу отказов выше); 12 лишних объектов удалены. `upload_compensation.py` после исправления проверяет компенсацию при ошибке БД и очистку janitor-ом после падения процесса. + +### Отказы + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +|-----------------------------------------------------------------|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| +| `docker stop` обслужившей API-реплики (×2 + для каждой реплики) | запись и download через другую | 0 ошибок записи; файл скачан по SHA-256 через оставшуюся | первая успешная запись через 4.0–4.2 s; потерь нет | +| Restart lb + api + db + valkey + s3 с volumes | данные, сессия и оригинал сохранены | PASS с прежним токеном | hook и ожидание готовности уложились в лимиты теста (90 + 90 s), длительность отдельно не замерялась; потерь нет | +| `docker stop s3` | только вложения → 503, остальное работает | ready 200, message 201, metadata 200, upload/download 503 за ~6 s | сразу после старта S3; потерь нет, intents неудачных PUT убирает janitor | +| Падение процесса после PUT до COMMIT | клиент без 201, orphan не вечный | intent + объект → удалены janitor после grace | grace (900 s; 20 s в тесте) | +| Потеря БД после PUT | 503, без мусора | 503; фоновая компенсация удалила объект и intent | ~0.1 s | +| 12 параллельных PUT по 10 MiB при лимите S3 512 MiB | — | SeaweedFS убит по памяти, все 503 | исправлено лимитом 1 GiB + `GOMEMLIMIT` | +| Оборванная клиентом загрузка | реплика продолжает работу | следующая загрузка 201; по коду PUT начинается только после приёма всего тела, поэтому объекта нет (отдельно не измерялось) | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker host, один PostgreSQL, один SeaweedFS без репликации, один LB: выдерживается отказ одной API-реплики и перезапуск хранилищ с volumes, но не потеря host/диска/S3-тома. Резервного копирования нет. +- Нет отдельного reconciler: объект без строки и без intent (ручное вмешательство, баг) автоматически не удаляется; потеря байтов готового вложения видна только по 500 и ERROR в логе при download. +- Прогрессивные JPEG > 16 Мпикс проверяются только по структуре маркеров (полное декодирование держало бы все DCT-коэффициенты в памяти); такие повреждения внутри энтропийного потока не ловятся. +- `MAX_CONCURRENT_UPLOADS` — на реплику, не глобально; общий предел S3 задаётся числом реплик × 8. Ёмкость SeaweedFS при большем числе реплик не измерялась. +- Измерения памяти — по одному-два прогона на ноутбуке; при N=12 в разных прогонах прирост был от 3.6 до 13.7 MiB на реплику, абсолютные числа зависят от аллокатора и версии Python. diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 00000000..e45cd2c4 --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,441 @@ +"""Attachments: synchronous upload to S3, metadata in PostgreSQL (lab4). + +Upload, in this order (all checks before the bytes are stored): + + 1. filename (400) -> chat membership (404) -> Content-Type header (415) + -> Content-Length (413 without reading the body); + 2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429; + 3. the body is read in chunks into a SpooledTemporaryFile (only + UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more + than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400; + 4. content validation in a thread (app.images): format by magic bytes, + pixel limit from the header (413), integrity (415); + 5. upload intent COMMIT -> S3 PUT -> one transaction "delete intent + + insert ready attachment" -> 201. + +Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs +in the background): + - PUT failed or timed out: the outcome is unknown - S3 may still finish + writing the object AFTER our DELETE. So the object is deleted best + effort, but the intent is KEPT; the janitor deletes the object again + after UPLOAD_GRACE_SECONDS and only then the intent; + - COMMIT failed: the outcome is unknown too - COMMIT may have succeeded + and only the answer was lost. Under the intent's row lock: no intent -> + the row is committed, keep the object; else delete object, then intent; + - the process died after PUT: the intent stays; the janitor deletes the + object and the intent after UPLOAD_GRACE_SECONDS. +So an object without a row exists only while its intent exists, and an +object of a committed row is never deleted. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import random +import tempfile +import unicodedata +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote +from uuid import UUID, uuid4 + +from fastapi import Request +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.requests import ClientDisconnect + +from app import repositories +from app.config import settings +from app.db import SessionLocal +from app.errors import ( + ApiError, + InvalidRequest, + NotFound, + PayloadTooLarge, + RateLimited, + Unavailable, + UnsupportedMediaType, +) +from app.images import InvalidImage, TooManyPixels, validate_image +from app.storage import ObjectMissing, StorageUnavailable, storage + + +logger = logging.getLogger("messenger.attachments") + +ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"} +# Characters that can turn a display name into a path or hide its real +# extension: separators, NUL/control characters, bidi overrides etc. +FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"} + +upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads) +# Strong references to background compensation tasks (asyncio keeps only +# weak ones); if the process dies first, the intent stays for the janitor. +_compensations: set[asyncio.Task] = set() + + +def _to_attachment(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def validate_filename(filename: str) -> str: + if not 1 <= len(filename) <= 255 or not filename.strip(): + raise InvalidRequest("filename must be 1-255 characters, not blank.") + + if "/" in filename or "\\" in filename: + raise InvalidRequest("filename must not contain path separators.") + + if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename): + raise InvalidRequest("filename must not contain control characters.") + + if filename.strip(". ") == "": + raise InvalidRequest("filename must not be a relative path.") + + return filename + + +def declared_content_type(request: Request) -> str: + raw = request.headers.get("Content-Type", "") + media_type = raw.split(";", 1)[0].strip().lower() + + if media_type not in ALLOWED_CONTENT_TYPES: + raise UnsupportedMediaType( + "Content-Type must be image/png or image/jpeg (raw bytes, not multipart)." + ) + + return media_type + + +def _too_large(message: str) -> PayloadTooLarge: + error = PayloadTooLarge(message) + # The rest of the body is not read: the connection cannot be reused. + error.headers["Connection"] = "close" + return error + + +def check_declared_length(request: Request) -> None: + raw = request.headers.get("Content-Length") + if raw is None: + return # chunked: the limit is enforced while reading + + if not raw.isdigit(): + raise InvalidRequest("Invalid Content-Length.") + + if int(raw) > settings.max_upload_bytes: + raise _too_large("File is larger than 10 MiB.") + + +async def _receive_body(request: Request, spool) -> tuple[int, str]: + """Stream the body into the spool; return (size, sha256).""" + digest = hashlib.sha256() + size = 0 + + try: + async with asyncio.timeout(settings.upload_timeout_seconds): + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > settings.max_upload_bytes: + raise _too_large("File is larger than 10 MiB.") + digest.update(chunk) + spool.write(chunk) + except TimeoutError: + raise ApiError( + 400, + "invalid_request", + "Upload was not completed in time.", + {"Connection": "close"}, + ) from None + except ClientDisconnect: + # Nobody will read this answer; nothing was stored anywhere. + logger.info("upload interrupted by the client after %d bytes", size) + raise InvalidRequest("Upload was interrupted.") from None + + return size, digest.hexdigest() + + +async def _acquire_slot() -> None: + try: + async with asyncio.timeout(settings.upload_slot_wait_seconds): + await upload_slots.acquire() + except TimeoutError: + raise RateLimited("Too many concurrent uploads, retry later.", 2) from None + + +async def _discard_after_failed_put(object_key: str) -> None: + """Best effort only; the intent stays for the janitor (late PUT).""" + try: + await storage.delete(object_key) + except StorageUnavailable: + logger.warning("compensation: S3 delete failed, janitor will retry") + + +async def _compensate_failed_commit(object_key: str) -> None: + """Delete the object only if its upload really did not commit.""" + try: + async with SessionLocal() as db: + row = await repositories.lock_upload_intent(db, object_key) + if row is None: + # COMMIT succeeded after all (or the janitor finished it). + await db.rollback() + return + if not row["committed"]: + try: + await storage.delete(object_key) + except StorageUnavailable: + await db.rollback() + logger.warning("compensation: S3 delete failed, janitor will retry") + return + await repositories.delete_upload_intent(db, object_key) + except Exception as exc: + logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__) + + +async def upload_attachment( + request: Request, + db: AsyncSession, + *, + chat_id: UUID, + filename: str, + current_user: dict[str, Any], +) -> dict[str, Any]: + validate_filename(filename) + + chat = await repositories.get_accessible_chat( + db, chat_id=chat_id, user_id=current_user["id"] + ) + if chat is None: + raise NotFound() + # Do not keep a pooled connection idle in a transaction while the + # client uploads up to 10 MiB. + await db.rollback() + + content_type = declared_content_type(request) + check_declared_length(request) + + await _acquire_slot() + try: + with tempfile.SpooledTemporaryFile( + max_size=settings.upload_spool_memory_bytes + ) as spool: + size, sha256 = await _receive_body(request, spool) + if size == 0: + raise InvalidRequest("Request body with image bytes is required.") + + try: + info = await asyncio.to_thread( + validate_image, spool, content_type, settings.max_image_pixels + ) + except TooManyPixels: + raise PayloadTooLarge("Image has more than 40 million pixels.") from None + except InvalidImage as exc: + raise UnsupportedMediaType(str(exc)) from None + + return await _store( + db, + spool=spool, + chat_id=chat_id, + owner_id=current_user["id"], + filename=filename, + content_type=info.content_type, + size=size, + sha256=sha256, + width=info.width, + height=info.height, + ) + finally: + upload_slots.release() + + +async def _store( + db: AsyncSession, + *, + spool, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size: int, + sha256: str, + width: int, + height: int, +) -> dict[str, Any]: + attachment_id = uuid4() + # Server-generated random key, unrelated to the filename and to the + # public attachment ID. + object_key = f"{settings.s3_object_prefix}{uuid4().hex}" + + await repositories.create_upload_intent( + db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id + ) + + try: + await storage.put(object_key, spool, size, content_type) + except StorageUnavailable as exc: + logger.warning("S3 PUT failed: %s", exc) + _in_background(_discard_after_failed_put(object_key)) + raise Unavailable() from None + + if settings.fault_injection == "crash_after_s3_put": + logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT") + os._exit(1) + + try: + if settings.fault_injection == "db_error_after_s3_put": + raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT") + + row = await repositories.finalize_attachment( + db, + attachment_id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + content_type=content_type, + size_bytes=size, + sha256=sha256, + width=width, + height=height, + object_key=object_key, + ) + except Exception as exc: + logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__) + _in_background(_compensate_failed_commit(object_key)) + raise Unavailable() from None + + if row is None: + # The janitor already claimed this intent (upload slower than the + # grace period): the object is (being) deleted, report a failure. + logger.warning("upload intent expired before COMMIT") + raise Unavailable() + + return _to_attachment(row) + + +async def get_attachment( + db: AsyncSession, + *, + attachment_id: UUID, + current_user: dict[str, Any], +) -> dict[str, Any]: + row = await repositories.get_accessible_attachment( + db, attachment_id=attachment_id, user_id=current_user["id"] + ) + if row is None: + raise NotFound() + return _to_attachment(row) + + +async def open_attachment_content( + db: AsyncSession, + *, + attachment_id: UUID, + variant: str, + current_user: dict[str, Any], +) -> tuple[AsyncIterator[bytes], dict[str, str]]: + """Authorize in PostgreSQL, then stream the bytes from S3 (no redirect).""" + row = await repositories.get_accessible_attachment( + db, attachment_id=attachment_id, user_id=current_user["id"] + ) + if row is None: + raise NotFound() + await db.rollback() + + if variant != "original": + # Lab4 has no variants; a ready attachment without it -> 404. + raise NotFound() + + try: + chunks, length = await storage.open_download(row["object_key"]) + except StorageUnavailable as exc: + logger.warning("S3 GET failed: %s", exc) + raise Unavailable() from None + except ObjectMissing: + # Metadata says "ready" but the bytes are gone: an inconsistency, + # not a user error (logged as ERROR). + logger.error("object of attachment %s is missing in S3", attachment_id) + raise + + if length != row["size_bytes"]: + logger.error("object of attachment %s has a wrong size", attachment_id) + raise ObjectMissing(row["object_key"]) + + headers = { + "Content-Type": row["content_type"], + "Content-Length": str(length), + "Content-Disposition": "inline; filename*=UTF-8''" + + quote(row["filename"], safe=""), + "ETag": f'"{row["sha256"]}"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + } + return chunks, headers + + +def _in_background(compensation) -> None: + """Do not keep the client waiting for compensation: with S3 down each + attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503 + anyway.""" + task = asyncio.create_task(compensation) + _compensations.add(task) + task.add_done_callback(_compensations.discard) + + +# --- Janitor ---------------------------------------------------------------- + +async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int: + """Delete objects of upload intents older than the grace period. + + One intent per transaction: lock it (SKIP LOCKED), delete the object in + S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and + stop: the intent stays and the next run retries. + """ + cleaned = 0 + for _ in range(limit): + async with SessionLocal() as db: + row = await repositories.lock_stale_intent(db, grace_seconds) + if row is None: + await db.rollback() + break + + if not row["committed"]: + try: + await storage.delete(row["object_key"]) + except StorageUnavailable: + await db.rollback() + logger.warning("janitor: S3 unavailable, will retry") + break + + await repositories.delete_upload_intent(db, row["object_key"]) + cleaned += 1 + logger.warning( + "janitor: removed abandoned upload %s (object %s)", + row["object_key"], + "kept, row committed" if row["committed"] else "deleted", + ) + return cleaned + + +async def janitor_loop() -> None: + while True: + # Jitter: replicas do not all wake up at the same moment. + await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2)) + try: + await cleanup_stale_uploads(settings.upload_grace_seconds) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("janitor run failed: %s", type(exc).__name__) diff --git a/app/config.py b/app/config.py index dc24312f..942941ea 100644 --- a/app/config.py +++ b/app/config.py @@ -63,6 +63,39 @@ class Settings: # for in-flight requests up to shutdown_timeout_seconds. shutdown_drain_seconds: float shutdown_timeout_seconds: int + # Lab4: attachments. Metadata lives in PostgreSQL, bytes in a private + # S3 bucket; the API uses its own restricted credentials (not root). + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + s3_object_prefix: str + s3_connect_timeout_seconds: float + s3_read_timeout_seconds: float + # Deadline of one whole S3 call (DNS + retries included), below the LB + # "timeout server" (10 s): a stopped S3 container makes Docker DNS hang + # ~8 s per lookup, which connect_timeout does not cover. + s3_operation_timeout_seconds: float + # PUT gets operation timeout + size / this rate: 10 MiB under load is + # slower than a HEAD. HAProxy allows 40 s for an upload POST. + s3_put_min_bytes_per_second: int + max_upload_bytes: int + max_image_pixels: int + # One upload may not take unlimited RAM/time: the body is spooled to a + # temporary file (only upload_spool_memory_bytes stay in RAM), receiving + # it is limited by upload_timeout_seconds, and at most + # max_concurrent_uploads uploads per replica run at once. + upload_spool_memory_bytes: int + upload_timeout_seconds: float + max_concurrent_uploads: int + upload_slot_wait_seconds: float + # Janitor: an upload intent older than upload_grace_seconds means the + # request died between S3 PUT and COMMIT; its object is deleted. + upload_grace_seconds: int + janitor_interval_seconds: float + # Experiments only (see REPORT.md): "" | crash_after_s3_put | db_error_after_s3_put + fault_injection: str def load_settings() -> Settings: @@ -106,6 +139,41 @@ def load_settings() -> Settings: shutdown_timeout_seconds=read_positive_int( "SHUTDOWN_TIMEOUT_SECONDS", "8" ), + s3_endpoint=read_value("S3_ENDPOINT", "http://s3:9000"), + s3_region=os.getenv("S3_REGION", "us-east-1"), + s3_bucket=read_value("S3_BUCKET", "messenger-attachments"), + s3_access_key_id=read_value("S3_ACCESS_KEY_ID"), + s3_secret_access_key=read_value("S3_SECRET_ACCESS_KEY"), + s3_object_prefix=os.getenv("S3_OBJECT_PREFIX", "originals/"), + s3_connect_timeout_seconds=read_positive_float( + "S3_CONNECT_TIMEOUT_SECONDS", "2" + ), + s3_read_timeout_seconds=read_positive_float( + "S3_READ_TIMEOUT_SECONDS", "5" + ), + s3_operation_timeout_seconds=read_positive_float( + "S3_OPERATION_TIMEOUT_SECONDS", "6" + ), + s3_put_min_bytes_per_second=read_positive_int( + "S3_PUT_MIN_BYTES_PER_SECOND", "1048576" + ), + max_upload_bytes=read_positive_int("MAX_UPLOAD_BYTES", "10485760"), + max_image_pixels=read_positive_int("MAX_IMAGE_PIXELS", "40000000"), + upload_spool_memory_bytes=read_positive_int( + "UPLOAD_SPOOL_MEMORY_BYTES", "1048576" + ), + upload_timeout_seconds=read_positive_float( + "UPLOAD_TIMEOUT_SECONDS", "60" + ), + max_concurrent_uploads=read_positive_int("MAX_CONCURRENT_UPLOADS", "8"), + upload_slot_wait_seconds=read_positive_float( + "UPLOAD_SLOT_WAIT_SECONDS", "10" + ), + upload_grace_seconds=read_positive_int("UPLOAD_GRACE_SECONDS", "900"), + janitor_interval_seconds=read_positive_float( + "JANITOR_INTERVAL_SECONDS", "60" + ), + fault_injection=os.getenv("FAULT_INJECTION", ""), ) diff --git a/app/errors.py b/app/errors.py index 171182b7..4e57091d 100644 --- a/app/errors.py +++ b/app/errors.py @@ -55,6 +55,29 @@ class Unavailable(ApiError): def __init__(self) -> None: super().__init__(503, "unavailable", "Service is unavailable.") + +class PayloadTooLarge(ApiError): + def __init__(self, message: str = "Payload is too large.") -> None: + super().__init__(413, "payload_too_large", message) + + +class UnsupportedMediaType(ApiError): + def __init__( + self, + message: str = "Only PNG and JPEG images are accepted.", + ) -> None: + super().__init__(415, "unsupported_media_type", message) + + +class RateLimited(ApiError): + def __init__(self, message: str, retry_after_seconds: int = 1) -> None: + super().__init__( + 429, + "rate_limited", + message, + {"Retry-After": str(retry_after_seconds)}, + ) + # X-Request-Id from HAProxy (or the client) is reused only if it is short and # printable; anything else is replaced, so it cannot inject into logs. REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}") diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..1e337088 --- /dev/null +++ b/app/images.py @@ -0,0 +1,264 @@ +"""Content-based validation of uploaded PNG/JPEG images. + +The declared Content-Type is never trusted: the format is detected from the +first bytes and must match it. Dimensions are read from the header and the +pixel limit is applied BEFORE any decoding, so a small "pixel bomb" file +cannot make the API allocate gigabytes. + +- PNG is validated by our own streaming reader: chunk lengths, CRC of every + chunk, IHDR, and the IDAT zlib stream is decompressed in 64 KiB pieces + that are counted and thrown away. Memory stays O(64 KiB) for any image. +- JPEG: Pillow reads the header (lazy open); then the image is decoded with + draft() at 1/8 scale, which makes libjpeg read the whole entropy-coded + stream (a truncated/corrupted file raises) while allocating only 1/64 of + the full-size pixel buffer. Large progressive JPEGs keep all DCT + coefficients in memory even with draft(), so above + PROGRESSIVE_DECODE_MAX_PIXELS only the marker structure is checked. + +The functions are synchronous (CPU work); call them via asyncio.to_thread. +""" +from __future__ import annotations + +import struct +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image + +# Our own limit is applied before Pillow decodes anything. +Image.MAX_IMAGE_PIXELS = None + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" +CHUNK = 64 * 1024 +PROGRESSIVE_DECODE_MAX_PIXELS = 16_000_000 + + +class InvalidImage(Exception): + """Not a PNG/JPEG, a corrupted file or a Content-Type mismatch (-> 415).""" + + +class TooManyPixels(Exception): + """Decoded image would exceed MAX_IMAGE_PIXELS (-> 413).""" + + +@dataclass(frozen=True) +class ImageInfo: + content_type: str + width: int + height: int + + +def sniff_content_type(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return "image/png" + if head.startswith(JPEG_SIGNATURE): + return "image/jpeg" + return None + + +def validate_image( + file: BinaryIO, + declared_content_type: str, + max_pixels: int, +) -> ImageInfo: + file.seek(0) + detected = sniff_content_type(file.read(len(PNG_SIGNATURE))) + file.seek(0) + + if detected is None: + raise InvalidImage("Content is not a PNG or JPEG image.") + + if detected != declared_content_type: + raise InvalidImage("Content-Type does not match the image content.") + + if detected == "image/png": + width, height = _validate_png(file, max_pixels) + else: + width, height = _validate_jpeg(file, max_pixels) + + return ImageInfo(detected, width, height) + + +# --- PNG ------------------------------------------------------------------- + +# color type -> (channels, allowed bit depths) +_PNG_COLOR_TYPES = { + 0: (1, {1, 2, 4, 8, 16}), + 2: (3, {8, 16}), + 3: (1, {1, 2, 4, 8}), + 4: (2, {8, 16}), + 6: (4, {8, 16}), +} +_ADAM7_PASSES = ( + (0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), + (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2), +) + + +def _read_exact(file: BinaryIO, size: int) -> bytes: + data = file.read(size) + if len(data) != size: + raise InvalidImage("Image file is truncated.") + return data + + +def _png_raw_size(width: int, height: int, bits_per_pixel: int, interlace: int) -> int: + """Size of the decompressed IDAT stream: filter byte + packed row bytes.""" + def scanlines(w: int, h: int) -> int: + if w == 0 or h == 0: + return 0 + return h * (1 + (w * bits_per_pixel + 7) // 8) + + if interlace == 0: + return scanlines(width, height) + + total = 0 + for x0, y0, dx, dy in _ADAM7_PASSES: + pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0 + pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0 + total += scanlines(pass_width, pass_height) + return total + + +def _validate_png(file: BinaryIO, max_pixels: int) -> tuple[int, int]: + _read_exact(file, len(PNG_SIGNATURE)) + + width = height = 0 + expected_raw = 0 + produced_raw = 0 + inflater = zlib.decompressobj() + color_type = -1 + seen_plte = False + idat_state = "before" # before -> inside -> after + first = True + + while True: + header = _read_exact(file, 8) + length, chunk_type = struct.unpack(">I4s", header) + + if length > 0x7FFFFFFF: + raise InvalidImage("Invalid PNG chunk length.") + + if first and chunk_type != b"IHDR": + raise InvalidImage("PNG must start with IHDR.") + + crc = zlib.crc32(chunk_type) + is_idat = chunk_type == b"IDAT" + + if is_idat: + if idat_state == "after": + raise InvalidImage("PNG IDAT chunks are not consecutive.") + if color_type == 3 and not seen_plte: + raise InvalidImage("PNG palette image without PLTE.") + idat_state = "inside" + elif idat_state == "inside": + idat_state = "after" + + data = b"" + remaining = length + while remaining: + piece = _read_exact(file, min(CHUNK, remaining)) + remaining -= len(piece) + crc = zlib.crc32(piece, crc) + + if is_idat: + produced_raw += _inflate_counting( + inflater, piece, expected_raw - produced_raw + ) + elif chunk_type == b"IHDR": + data += piece + + (stored_crc,) = struct.unpack(">I", _read_exact(file, 4)) + if stored_crc != crc: + raise InvalidImage("PNG chunk CRC mismatch.") + + if first: + first = False + if length != 13: + raise InvalidImage("Invalid PNG IHDR.") + (width, height, bit_depth, color_type, + compression, filter_method, interlace) = struct.unpack(">IIBBBBB", data) + + if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise InvalidImage("Invalid PNG dimensions.") + if width * height > max_pixels: + # Before a single IDAT byte is decompressed. + raise TooManyPixels() + if color_type not in _PNG_COLOR_TYPES: + raise InvalidImage("Invalid PNG color type.") + channels, depths = _PNG_COLOR_TYPES[color_type] + if bit_depth not in depths or compression != 0 or filter_method != 0 \ + or interlace not in (0, 1): + raise InvalidImage("Invalid PNG IHDR.") + expected_raw = _png_raw_size(width, height, channels * bit_depth, interlace) + elif chunk_type == b"PLTE": + seen_plte = True + elif chunk_type == b"IEND": + break + + if idat_state == "before": + raise InvalidImage("PNG has no image data.") + + produced_raw += len(inflater.flush()) + if not inflater.eof or produced_raw != expected_raw: + raise InvalidImage("PNG image data is corrupted or truncated.") + + return width, height + + +def _inflate_counting(inflater, data: bytes, allowed: int) -> int: + """Decompress and discard; return the number of produced bytes.""" + produced = 0 + try: + while data: + if inflater.eof: + raise InvalidImage("PNG has data after the end of the zlib stream.") + out = inflater.decompress(data, CHUNK) + produced += len(out) + if produced > allowed: + raise InvalidImage("PNG image data is larger than IHDR declares.") + data = inflater.unconsumed_tail + except zlib.error: + raise InvalidImage("PNG image data is corrupted.") from None + return produced + + +# --- JPEG ------------------------------------------------------------------ + +def _validate_jpeg(file: BinaryIO, max_pixels: int) -> tuple[int, int]: + try: + with Image.open(file, formats=["JPEG"]) as image: + width, height = image.size + + if width < 1 or height < 1: + raise InvalidImage("Invalid JPEG dimensions.") + if width * height > max_pixels: + # Header only: no pixel has been decoded yet. + raise TooManyPixels() + + progressive = bool( + image.info.get("progressive") or image.info.get("progression") + ) + + if progressive and width * height > PROGRESSIVE_DECODE_MAX_PIXELS: + _check_jpeg_end(file) + else: + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + except (InvalidImage, TooManyPixels): + raise + except Exception: + # Pillow raises OSError/SyntaxError/ValueError/... for broken data. + raise InvalidImage("JPEG image is corrupted or truncated.") from None + + return width, height + + +def _check_jpeg_end(file: BinaryIO) -> None: + file.seek(0, 2) + size = file.tell() + file.seek(max(0, size - 2)) + if file.read(2) != b"\xff\xd9": + raise InvalidImage("JPEG image is truncated.") diff --git a/app/main.py b/app/main.py index 378c7203..a1e2efd7 100644 --- a/app/main.py +++ b/app/main.py @@ -1,17 +1,26 @@ from __future__ import annotations +import asyncio import logging from collections.abc import AsyncIterator from contextlib import asynccontextmanager from uuid import UUID -from fastapi import Depends, FastAPI, Header, Request, Response, Security +from typing import Literal + +from fastapi import Depends, FastAPI, Header, Query, Request, Response, Security from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from sqlalchemy import exc as sa_exc from starlette.exceptions import HTTPException as StarletteHTTPException from sqlalchemy.ext.asyncio import AsyncSession +from app.attachments import ( + get_attachment, + janitor_loop, + open_attachment_content, + upload_attachment, +) from app.cache import session_cache from app.db import check_database_ready, engine, get_db from app.deps import basic_scheme, pagination_params, require_user @@ -25,6 +34,7 @@ from app.errors import ( ) from app.lifecycle import draining from app.models import ( + Attachment, Chat, ChatPage, CreateChat, @@ -36,6 +46,7 @@ from app.models import ( Session, User, ) +from app.storage import storage from app.services import ( create_chat, create_message, @@ -50,14 +61,19 @@ from app.services import ( @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: + await storage.start() + janitor = asyncio.create_task(janitor_loop()) yield + janitor.cancel() + await asyncio.gather(janitor, return_exceptions=True) + await storage.close() await session_cache.close() await engine.dispose() app = FastAPI( - title="MTUSI Messenger — лабораторная 3", - version="1.3.0", + title="MTUSI Messenger — лабораторная 4", + version="1.4.0", lifespan=lifespan, ) @@ -185,6 +201,9 @@ async def liveness() -> dict[str, str]: async def readiness() -> dict[str, str]: # A draining replica reports "not ready" so HAProxy stops sending new # requests to it before the listener is closed. + # S3 is deliberately NOT checked: without it only attachments degrade + # (503), users/chats/messages keep working. If readiness depended on S3, + # an S3 outage would take every replica out of the LB. if draining.is_set() or not await check_database_ready(): raise Unavailable() @@ -331,6 +350,7 @@ async def create_message_route( db, chat_id=chat_id, text_value=payload.text, + attachment_ids=payload.attachment_ids, idempotency_key=idempotency_key, current_user=current_user, ) @@ -359,4 +379,82 @@ async def list_messages_route( limit=limit, cursor=cursor, current_user=current_user, - ) \ No newline at end of file + ) + + +UPLOAD_BODY = { + "requestBody": { + "required": True, + "description": "Raw image bytes, not multipart and not JSON/base64.", + "content": { + "image/png": {"schema": {"type": "string", "format": "binary"}}, + "image/jpeg": {"schema": {"type": "string", "format": "binary"}}, + }, + } +} + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + status_code=201, + response_model=Attachment, + openapi_extra=UPLOAD_BODY, +) +async def upload_attachment_route( + chat_id: UUID, + request: Request, + filename: str = Query(min_length=1, max_length=255), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + return await upload_attachment( + request, + db, + chat_id=chat_id, + filename=filename, + current_user=current_user, + ) + + +@app.get( + "/api/v1/attachments/{attachment_id}", + response_model=Attachment, +) +async def get_attachment_route( + attachment_id: UUID, + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + return await get_attachment( + db, + attachment_id=attachment_id, + current_user=current_user, + ) + + +@app.get( + "/api/v1/attachments/{attachment_id}/content", + response_class=StreamingResponse, + responses={200: {"content": {"image/png": {}, "image/jpeg": {}}}}, +) +async def download_attachment_route( + attachment_id: UUID, + # Known variant names are accepted so that a ready attachment without + # that variant answers 404 (contract); any other value is 400. + variant: Literal["original", "thumbnail", "crop", "optimized"] = Query( + default="original" + ), + current_user: dict = Depends(require_user), + db: AsyncSession = Depends(get_db), +): + chunks, headers = await open_attachment_content( + db, + attachment_id=attachment_id, + variant=variant, + current_user=current_user, + ) + return StreamingResponse( + chunks, + media_type=headers.pop("Content-Type"), + headers=headers, + ) diff --git a/app/models.py b/app/models.py index 9615aa3a..7b01367c 100644 --- a/app/models.py +++ b/app/models.py @@ -4,7 +4,7 @@ from datetime import datetime, timezone from typing import Literal from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class ApiModel(BaseModel): @@ -92,15 +92,26 @@ class ChatPage(ApiModel): class CreateMessage(ApiModel): - text: str = Field(min_length=1, max_length=4000) + # Lab4: text may be "" when the message carries attachments. + text: str = Field(max_length=4000) + # 0-10 unique IDs of attachments of the same chat; missing == []. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) - @field_validator("text") + @field_validator("attachment_ids") @classmethod - def text_is_not_blank(cls, value: str) -> str: - if not value.strip(): - raise ValueError("text must not contain only spaces.") + def attachment_ids_are_unique(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError("attachment_ids must be unique.") return value + @model_validator(mode="after") + def text_or_attachments(self) -> "CreateMessage": + if not self.attachment_ids and not self.text.strip(): + raise ValueError( + "text must not be blank when there are no attachments." + ) + return self + class Message(ApiModel): id: UUID @@ -108,6 +119,7 @@ class Message(ApiModel): sender_id: UUID text: str created_at: datetime + attachment_ids: list[UUID] = Field(default_factory=list) @field_validator("created_at") @classmethod @@ -136,6 +148,38 @@ class Session(ApiModel): return value.astimezone(timezone.utc) +class Variant(ApiModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(ApiModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int + sha256: str + # Lab4: synchronous upload, a stored attachment is always ready. + status: Literal["ready"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + @field_validator("created_at", "updated_at") + @classmethod + def attachment_time_is_utc(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("timestamps must include UTC timezone.") + return value.astimezone(timezone.utc) + + class Health(ApiModel): status: Literal["ok"] diff --git a/app/repositories.py b/app/repositories.py index 6a5a2d17..2aa1d664 100644 --- a/app/repositories.py +++ b/app/repositories.py @@ -1,4 +1,4 @@ -"""PostgreSQL repository functions for MTUSI Messenger Lab3.""" +"""PostgreSQL repository functions for MTUSI Messenger Lab4.""" from __future__ import annotations from collections.abc import Iterable @@ -301,12 +301,28 @@ async def list_chats_for_user( return chats +# Attachment IDs of a message in the order of the request. +MESSAGE_ATTACHMENT_IDS = """ + COALESCE( + (SELECT array_agg(ma.attachment_id ORDER BY ma.position) + FROM message_attachments AS ma + WHERE ma.message_id = m.id), + CAST('{}' AS uuid[]) + ) AS attachment_ids +""" + + +class AttachmentsNotFound(Exception): + """Some attachment_ids do not exist or belong to another chat (-> 404).""" + + async def create_message_idempotent( db: AsyncSession, *, chat_id: UUID, sender_id: UUID, text_value: str, + attachment_ids: list[UUID], idempotency_key: str, request_hash: str, ttl_seconds: int, @@ -316,10 +332,11 @@ async def create_message_idempotent( 1. INSERT the key row. Its primary key (user, chat, method, key) lives in the shared PostgreSQL, so a concurrent request with the same key on ANY replica blocks on this INSERT until the first transaction ends. - 2. If the key was claimed: INSERT the message, link it to the key and - COMMIT both together. A failure before COMMIT rolls back both, so a - transient error is never stored as the final answer and a retry with - the same key simply runs again. + 2. If the key was claimed: check that every attachment belongs to this + chat, INSERT the message and its attachment links, link the message + to the key and COMMIT everything together. A failure before COMMIT + (also AttachmentsNotFound) rolls back all of it, so an error is never + stored as the final answer and a retry with the same key runs again. 3. If the key already exists (committed by someone else): return the stored message, or None when the body differs (-> 409). @@ -351,6 +368,23 @@ async def create_message_idempotent( ) if claimed.first() is not None: + if attachment_ids: + # Unknown and foreign IDs look the same (404): the response + # does not reveal that an attachment of another chat exists. + found = await db.execute( + text( + """ + SELECT id FROM attachments + WHERE chat_id = :chat_id + AND id = ANY(CAST(:ids AS uuid[])) + FOR SHARE + """ + ), + {"chat_id": chat_id, "ids": attachment_ids}, + ) + if {row.id for row in found} != set(attachment_ids): + raise AttachmentsNotFound() + result = await db.execute( text( """ @@ -368,6 +402,25 @@ async def create_message_idempotent( ) message = dict(result.mappings().one()) + if attachment_ids: + await db.execute( + text( + """ + INSERT INTO message_attachments + (message_id, attachment_id, position) + VALUES (:message_id, :attachment_id, :position) + """ + ), + [ + { + "message_id": message["id"], + "attachment_id": attachment_id, + "position": position, + } + for position, attachment_id in enumerate(attachment_ids) + ], + ) + await db.execute( text( """ @@ -382,13 +435,15 @@ async def create_message_idempotent( {**key_params, "message_id": message["id"]}, ) await db.commit() + message["attachment_ids"] = list(attachment_ids) return message, False stored = await db.execute( text( - """ + f""" SELECT k.request_hash, - m.id, m.chat_id, m.sender_id, m.text, m.created_at + m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {MESSAGE_ATTACHMENT_IDS} FROM idempotency_keys AS k JOIN messages AS m ON m.id = k.message_id WHERE k.user_id = :user_id @@ -424,10 +479,11 @@ async def list_messages_keyset( cursor_created_at: datetime | None, cursor_id: UUID | None, ) -> list[dict[str, Any]]: - query = """ - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = :chat_id \ + query = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {MESSAGE_ATTACHMENT_IDS} + FROM messages AS m + WHERE m.chat_id = :chat_id """ params: dict[str, Any] = { "chat_id": chat_id, @@ -436,11 +492,212 @@ async def list_messages_keyset( if cursor_created_at is not None and cursor_id is not None: query += """ - AND (created_at, id) > (:cursor_created_at, :cursor_id) + AND (m.created_at, m.id) > (:cursor_created_at, :cursor_id) """ params["cursor_created_at"] = cursor_created_at params["cursor_id"] = cursor_id - query += " ORDER BY created_at ASC, id ASC LIMIT :limit_plus_one" + query += " ORDER BY m.created_at ASC, m.id ASC LIMIT :limit_plus_one" result = await db.execute(text(query), params) return [dict(row) for row in result.mappings().all()] + + +# --- Attachments ------------------------------------------------------------ + +ATTACHMENT_COLUMNS = """ + a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, + a.sha256, a.status, a.error_code, a.object_key, a.created_at, a.updated_at +""" + + +async def create_upload_intent( + db: AsyncSession, + *, + object_key: str, + attachment_id: UUID, + chat_id: UUID, +) -> None: + """Committed BEFORE the S3 PUT: from now on the object can be found and + cleaned up even if this process dies right after the PUT.""" + try: + await db.execute( + text( + """ + INSERT INTO upload_intents (object_key, attachment_id, chat_id) + VALUES (:object_key, :attachment_id, :chat_id) + """ + ), + { + "object_key": object_key, + "attachment_id": attachment_id, + "chat_id": chat_id, + }, + ) + await db.commit() + except Exception: + await db.rollback() + raise + + +async def finalize_attachment( + db: AsyncSession, + *, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + object_key: str, +) -> dict[str, Any] | None: + """Delete the intent and insert the ready attachment in ONE transaction. + + Returns None if the intent is already gone: the janitor decided this + upload was abandoned and deleted the object, so the row must not be + created. The DELETE waits on the janitor's row lock, so the two can + never both succeed. + """ + try: + claimed = await db.execute( + text( + "DELETE FROM upload_intents WHERE object_key = :object_key RETURNING 1" + ), + {"object_key": object_key}, + ) + if claimed.first() is None: + await db.rollback() + return None + + result = await db.execute( + text( + f""" + INSERT INTO attachments AS a + (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key) + VALUES (:id, :chat_id, :owner_id, :filename, :content_type, + :size_bytes, :sha256, :width, :height, :object_key) + RETURNING {ATTACHMENT_COLUMNS} + """ + ), + { + "id": attachment_id, + "chat_id": chat_id, + "owner_id": owner_id, + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "sha256": sha256, + "width": width, + "height": height, + "object_key": object_key, + }, + ) + row = dict(result.mappings().one()) + await db.commit() + return row + except Exception: + await db.rollback() + raise + + +async def delete_upload_intent(db: AsyncSession, object_key: str) -> None: + try: + await db.execute( + text("DELETE FROM upload_intents WHERE object_key = :object_key"), + {"object_key": object_key}, + ) + await db.commit() + except Exception: + await db.rollback() + raise + + +async def get_accessible_attachment( + db: AsyncSession, + *, + attachment_id: UUID, + user_id: UUID, +) -> dict[str, Any] | None: + """Only members of the attachment's chat see it; others get None (404).""" + result = await db.execute( + text( + f""" + SELECT {ATTACHMENT_COLUMNS} + FROM attachments AS a + JOIN chat_members AS cm + ON cm.chat_id = a.chat_id + AND cm.user_id = :user_id + WHERE a.id = :attachment_id + """ + ), + {"attachment_id": attachment_id, "user_id": user_id}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def lock_upload_intent( + db: AsyncSession, + object_key: str, +) -> dict[str, Any] | None: + """Lock the intent of one upload (waits for a finishing transaction); + the transaction stays open. None: no intent, nothing to compensate.""" + result = await db.execute( + text( + """ + SELECT i.object_key, + EXISTS (SELECT 1 FROM attachments AS a + WHERE a.object_key = i.object_key) AS committed + FROM upload_intents AS i + WHERE i.object_key = :object_key + FOR UPDATE OF i + """ + ), + {"object_key": object_key}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def lock_stale_intent( + db: AsyncSession, + grace_seconds: int, +) -> dict[str, Any] | None: + """Lock ONE intent older than the grace period; the transaction stays open. + + SKIP LOCKED: an upload that is finishing right now holds the row lock in + finalize_attachment and is skipped; several replicas can run the janitor + at once without touching the same intent. + """ + result = await db.execute( + text( + """ + SELECT i.object_key, + EXISTS (SELECT 1 FROM attachments AS a + WHERE a.object_key = i.object_key) AS committed + FROM upload_intents AS i + WHERE i.created_at < now() - make_interval(secs => :grace) + ORDER BY i.created_at + LIMIT 1 + FOR UPDATE OF i SKIP LOCKED + """ + ), + {"grace": grace_seconds}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +async def attachment_keys(db: AsyncSession) -> set[str]: + result = await db.execute(text("SELECT object_key FROM attachments")) + return {row.object_key for row in result} + + +async def intent_keys(db: AsyncSession) -> dict[str, datetime]: + result = await db.execute( + text("SELECT object_key, created_at FROM upload_intents") + ) + return {row.object_key: row.created_at for row in result} diff --git a/app/services.py b/app/services.py index 5a8b7433..8d293801 100644 --- a/app/services.py +++ b/app/services.py @@ -1,4 +1,4 @@ -"""Business rules for MTUSI Messenger Lab3.""" +"""Business rules for MTUSI Messenger Lab4 (users, chats, messages).""" from __future__ import annotations import asyncio @@ -58,6 +58,7 @@ def _to_message(row: dict[str, Any]) -> dict[str, Any]: "sender_id": row["sender_id"], "text": row["text"], "created_at": row["created_at"], + "attachment_ids": list(row.get("attachment_ids") or []), } @@ -299,10 +300,15 @@ async def list_chats( } -def _request_fingerprint(text_value: str) -> str: - """Hash of the validated body: key order and whitespace do not matter.""" +def _request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str: + """Hash of the validated body: key order and whitespace do not matter, + a missing attachment_ids equals []. Without attachments the canonical + form is the lab3 one, so keys stored by lab3 still replay correctly.""" + body: dict[str, Any] = {"text": text_value} + if attachment_ids: + body["attachment_ids"] = [str(item) for item in attachment_ids] canonical = json.dumps( - {"text": text_value}, + body, ensure_ascii=False, separators=(",", ":"), sort_keys=True, @@ -315,6 +321,7 @@ async def create_message( *, chat_id: UUID, text_value: str, + attachment_ids: list[UUID], idempotency_key: str | None, current_user: dict[str, Any], ) -> tuple[dict[str, Any], bool]: @@ -339,15 +346,19 @@ async def create_message( if chat is None: raise _not_found() - result = await repositories.create_message_idempotent( - db, - chat_id=chat_id, - sender_id=current_user["id"], - text_value=text_value, - idempotency_key=idempotency_key, - request_hash=_request_fingerprint(text_value), - ttl_seconds=settings.idempotency_ttl_seconds, - ) + try: + result = await repositories.create_message_idempotent( + db, + chat_id=chat_id, + sender_id=current_user["id"], + text_value=text_value, + attachment_ids=attachment_ids, + idempotency_key=idempotency_key, + request_hash=_request_fingerprint(text_value, attachment_ids), + ttl_seconds=settings.idempotency_ttl_seconds, + ) + except repositories.AttachmentsNotFound: + raise _not_found() from None if result is None: raise ApiError( diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..18a457b6 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,172 @@ +"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway). + +One long-lived aiobotocore client per API process, path-style addressing, +short connect/read timeouts, at most 2 attempts and an overall deadline +S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a +fast 503 on the attachment endpoints only. Any client-side or +network error is raised as StorageUnavailable; a missing key is +ObjectMissing. Object keys are generated by the server (see services.py); +the user's filename never becomes part of a key. +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import Any, BinaryIO + +from aiobotocore.config import AioConfig +from aiobotocore.session import get_session +from botocore.exceptions import BotoCoreError, ClientError + +from app.config import settings + + +logger = logging.getLogger("messenger.storage") + +DOWNLOAD_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 is unreachable, timed out or refused the request.""" + + +class ObjectMissing(Exception): + """The key does not exist in the bucket.""" + + +@dataclass +class StoredObject: + key: str + size: int + last_modified: Any + + +class ObjectStorage: + def __init__(self) -> None: + self._stack: AsyncExitStack | None = None + self._client = None + self.bucket = settings.s3_bucket + + async def start(self) -> None: + self._stack = AsyncExitStack() + session = get_session() + self._client = await self._stack.enter_async_context( + session.create_client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id, + aws_secret_access_key=settings.s3_secret_access_key, + config=AioConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + retries={"max_attempts": 2, "mode": "standard"}, + max_pool_connections=settings.max_concurrent_uploads * 2 + 4, + ), + ) + ) + + async def close(self) -> None: + if self._stack is not None: + await self._stack.aclose() + self._stack = None + self._client = None + + @staticmethod + def _deadline(size: int = 0): + return asyncio.timeout( + settings.s3_operation_timeout_seconds + + size / settings.s3_put_min_bytes_per_second + ) + + def _require_client(self): + if self._client is None: + raise StorageUnavailable("S3 client is not started") + return self._client + + async def put( + self, + key: str, + body: BinaryIO, + size: int, + content_type: str, + ) -> None: + """Upload a seekable file. SigV4 signs the payload SHA-256, so S3 + itself rejects bytes that were damaged on the way.""" + client = self._require_client() + body.seek(0) + try: + async with self._deadline(size): + await client.put_object( + Bucket=self.bucket, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ) + except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]: + """GET the object; S3 errors surface here, before any byte is sent.""" + client = self._require_client() + try: + async with self._deadline(): + response = await client.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: + raise ObjectMissing(key) from exc + raise StorageUnavailable(type(exc).__name__) from exc + except (BotoCoreError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + body = response["Body"] + + async def chunks() -> AsyncIterator[bytes]: + try: + async for chunk in body.iter_chunks(DOWNLOAD_CHUNK): + yield chunk + finally: + body.close() + + return chunks(), int(response["ContentLength"]) + + async def delete(self, key: str) -> None: + """Idempotent: deleting a missing key succeeds in S3.""" + client = self._require_client() + try: + async with self._deadline(): + await client.delete_object(Bucket=self.bucket, Key=key) + except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + async def exists(self, key: str) -> bool: + client = self._require_client() + try: + async with self._deadline(): + await client.head_object(Bucket=self.bucket, Key=key) + return True + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}: + return False + raise StorageUnavailable(type(exc).__name__) from exc + except (BotoCoreError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]: + client = self._require_client() + paginator = client.get_paginator("list_objects_v2") + try: + async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): + for item in page.get("Contents", []): + yield StoredObject(item["Key"], item["Size"], item["LastModified"]) + except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc: + raise StorageUnavailable(type(exc).__name__) from exc + + +storage = ObjectStorage() diff --git a/compose.example.yaml b/compose.example.yaml deleted file mode 100644 index 76a785ff..00000000 --- a/compose.example.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Topology worksheet, deliberately NOT a runnable stack (no application supplied). -# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment. -# Service names below are logical roles, not imposed container names. -x-lab-number: 4 -x-required-roles: - - api - - postgres - - load_balancer - - s3 -# TODO: implement real images/builds, network wiring, volumes, healthchecks, -# credentials and bootstrap. Details: contracts/infrastructure.json and README.md. -services: {} diff --git a/compose.yaml b/compose.yaml index 98dd4a1b..a5b64c95 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,8 +1,9 @@ #file: noinspection ComposeUnknownValues -# Lab3: client -> HAProxy -> API x N -> PostgreSQL (source of truth: data, -# sessions, idempotency keys) + Valkey (shared session cache from lab2). +# Lab4: client -> HAProxy -> API x N -> PostgreSQL (source of truth: data, +# sessions, idempotency keys, attachment metadata) + Valkey (session cache) +# + SeaweedFS S3 (private bucket with attachment bytes). # Start: see REPORT.md, "Запуск из чистого clone". -name: messenger-lab3 +name: messenger-lab4 x-api-env: &api-env APP_HOST: 0.0.0.0 @@ -23,6 +24,27 @@ x-api-env: &api-env IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}" SHUTDOWN_DRAIN_SECONDS: "3" SHUTDOWN_TIMEOUT_SECONDS: "8" + # Lab4: S3 with the API's own restricted credentials (not admin). + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_OBJECT_PREFIX: originals/ + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: "2" + S3_READ_TIMEOUT_SECONDS: "5" + S3_OPERATION_TIMEOUT_SECONDS: "6" + S3_PUT_MIN_BYTES_PER_SECOND: "1048576" + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + UPLOAD_SPOOL_MEMORY_BYTES: "1048576" + UPLOAD_TIMEOUT_SECONDS: "60" + MAX_CONCURRENT_UPLOADS: "8" + UPLOAD_SLOT_WAIT_SECONDS: "10" + UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}" + JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}" + # Experiments only: crash_after_s3_put | db_error_after_s3_put (REPORT.md). + FAULT_INJECTION: "${FAULT_INJECTION:-}" services: db: @@ -76,11 +98,71 @@ services: resources: limits: { cpus: "0.25", memory: 128M } + # S3: SeaweedFS all-in-one. No "ports": reachable only from the internal + # backend network; its admin web UIs are bound to 127.0.0.1 inside the + # container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume. + s3: + image: chrislusf/seaweedfs:3.97 + entrypoint: ["/bin/sh", "/opt/lab/start.sh"] + environment: + # Soft limit of the Go heap: without it the GC lets SeaweedFS grow to + # the container limit under parallel 10 MiB PUTs (measured, REPORT.md). + GOMEMLIMIT: 600MiB + volumes: + - s3_data:/data + - ./infra/seaweedfs:/opt/lab:ro + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_access_key_id + - s3_secret_access_key + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 60s + networks: [backend] + stop_grace_period: 30s + restart: unless-stopped + deploy: + resources: + # 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed + # SeaweedFS to the limit and the kernel killed it (REPORT.md). + limits: { cpus: "1.0", memory: 1G } + + # Idempotent bucket bootstrap + self-checks (private bucket, least + # privilege of the API key). The only container with the S3 admin key. + s3-init: + build: . + image: messenger-lab4-api:${APP_VERSION:-dev} + command: ["python", "scripts/s3_bootstrap.py"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_OBJECT_PREFIX: originals/ + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3: { condition: service_healthy } + networks: [backend] + restart: "no" + deploy: + resources: + limits: { cpus: "0.5", memory: 128M } + # Runs the migrations once, before any API replica starts (no race of # replicas). A repeated run only prints SKIP. migrate: build: . - image: messenger-lab3-api:${APP_VERSION:-dev} + image: messenger-lab4-api:${APP_VERSION:-dev} command: ["python", "scripts/migrate.py"] environment: DATABASE_URL_FILE: /run/secrets/database_url @@ -95,9 +177,9 @@ services: api: build: . - image: messenger-lab3-api:${APP_VERSION:-dev} + image: messenger-lab4-api:${APP_VERSION:-dev} environment: *api-env - secrets: [database_url, cache_url] + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] # No "ports": replicas are reachable only through the LB. # No "container_name": the service is scaled by deploy.replicas / --scale. networks: [edge, backend] @@ -105,6 +187,7 @@ services: db: { condition: service_healthy } valkey: { condition: service_healthy } migrate: { condition: service_completed_successfully } + s3-init: { condition: service_completed_successfully } healthcheck: test: - CMD @@ -157,20 +240,23 @@ services: working_dir: /work environment: BASE_URL: http://lb:8080 - COMPOSE_PROJECT: messenger-lab3 + COMPOSE_PROJECT: messenger-lab4 + S3_ENDPOINT: http://s3:9000 volumes: - ./:/work - /var/run/docker.sock:/var/run/docker.sock - networks: [edge] + # backend: only for the "private bucket from outside the API" check. + networks: [edge, backend] networks: edge: {} - # PostgreSQL and Valkey: no route outside, the LB is not attached. + # PostgreSQL, Valkey and S3: no route outside, the LB is not attached. backend: internal: true volumes: postgres_data: {} + s3_data: {} secrets: postgres_password: @@ -181,3 +267,11 @@ secrets: file: ./infra/secrets/valkey_password.txt cache_url: file: ./infra/secrets/cache_url.txt + s3_admin_access_key_id: + file: ./infra/secrets/s3_admin_access_key_id.txt + s3_admin_secret_access_key: + file: ./infra/secrets/s3_admin_secret_access_key.txt + s3_access_key_id: + file: ./infra/secrets/s3_access_key_id.txt + s3_secret_access_key: + file: ./infra/secrets/s3_secret_access_key.txt diff --git a/evidence/api-no-local-files.txt b/evidence/api-no-local-files.txt new file mode 100644 index 00000000..398734c3 --- /dev/null +++ b/evidence/api-no-local-files.txt @@ -0,0 +1,14 @@ +## f00b790a44e9 +mounts=[{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_access_key_id.txt","Destination":"/run/secrets/s3_access_key_id","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_secret_access_key.txt","Destination":"/run/secrets/s3_secret_access_key","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\database_url.txt","Destination":"/run/secrets/database_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\cache_url.txt","Destination":"/run/secrets/cache_url","Mode":"","RW":false,"Propagation":"rprivate"}] readonly_rootfs=false +tmp files: +total 8 +drwxrwxrwt 1 root root 4096 Oct 30 11:44 . +drwxr-xr-x 1 root root 4096 Oct 30 11:44 .. +files > 10KB written by appuser outside /proc,/sys: +## bb5edb449d7c +mounts=[{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\database_url.txt","Destination":"/run/secrets/database_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\cache_url.txt","Destination":"/run/secrets/cache_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_access_key_id.txt","Destination":"/run/secrets/s3_access_key_id","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_secret_access_key.txt","Destination":"/run/secrets/s3_secret_access_key","Mode":"","RW":false,"Propagation":"rprivate"}] readonly_rootfs=false +tmp files: +total 8 +drwxrwxrwt 2 root root 4096 Dec 2 2024 . +drwxr-xr-x 1 root root 4096 Oct 30 11:44 .. +files > 10KB written by appuser outside /proc,/sys: diff --git a/evidence/attachment-edge-cases.txt b/evidence/attachment-edge-cases.txt new file mode 100644 index 00000000..5e2e84b0 --- /dev/null +++ b/evidence/attachment-edge-cases.txt @@ -0,0 +1,63 @@ + Container messenger-lab4-tester-run-cb4971c6a617 Creating + Container messenger-lab4-tester-run-cb4971c6a617 Created +OK JPEG fixture -> 201 ready: 201 (ожидается 201 ) +OK участник скачивает JPEG побайтно, без redirect: 200 (ожидается 200 ) +OK PNG ровно 10 MiB -> 201: 201 (ожидается 201 ) +OK 10 MiB скачан побайтно: 200 (ожидается 200 ) +OK PNG ровно 40 млн пикселей -> 201: 201 (ожидается 201 ) +OK 10 MiB + 1 байт (Content-Length) -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK 11 MiB chunked без Content-Length -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK PNG-«бомба» 50 млн пикселей (~10 KB файл) -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK PNG 2^31-1 x 2^31-1 в IHDR -> 413: 413 payload_too_large (ожидается 413 payload_too_large) +OK пустое тело -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK PNG обрезан пополам -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG с неверным CRC -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG с мусором в zlib при верном CRC -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG без IEND -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK JPEG обрезан -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK JPEG только сигнатура -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK GIF как image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK случайные байты как image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK HTML как image/jpeg -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG под Content-Type image/jpeg -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK JPEG под Content-Type image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG под text/plain -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK PNG под image/gif -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK multipart/form-data -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK без Content-Type -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type) +OK filename ../../etc/passwd -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename C:\boot.png -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename NUL в имени -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename перевод строки -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename ESC-последовательность -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename bidi override (gnp.exe) -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename только пробелы -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename .. -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename 256 символов -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename пустой filename -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename нет filename -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK filename ровно 255 символов -> 201: 201 (ожидается 201 ) +OK upload без токена -> 401: 401 unauthorized (ожидается 401 unauthorized) +OK посторонний upload в чужой чат -> 404: 404 not_found (ожидается 404 not_found) +OK upload в несуществующий чат -> 404: 404 not_found (ожидается 404 not_found) +OK посторонний GET /attachments/id -> 404: 404 not_found (ожидается 404 not_found) +OK неизвестный attachment -> 404: 404 not_found (ожидается 404 not_found) +OK посторонний GET /attachments/id/content -> 404: 404 not_found (ожидается 404 not_found) +OK неизвестный attachment/content -> 404: 404 not_found (ожидается 404 not_found) +OK attachment_id не UUID -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK ready без варианта thumbnail -> 404: 404 not_found (ожидается 404 not_found) +OK variant неизвестного имени -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK вложение другого чата (своего) -> 404: 404 not_found (ожидается 404 not_found) +OK вложение чужого чата -> 404: 404 not_found (ожидается 404 not_found) +OK одно из ID неизвестно -> 404, ничего не создано: 404 not_found (ожидается 404 not_found) +OK дубли attachment_ids -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK 11 attachment_ids -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK attachment_id не UUID -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK пробельный text без вложений -> 400: 400 invalid_request (ожидается 400 invalid_request) +OK Bob: text="" + вложение Alice из общего чата -> 201: 201 (ожидается 201 ) +OK повтор Idempotency-Key -> то же сообщение: 201 (ожидается 201 ) +OK тот же ключ, другое тело -> 409: 409 conflict (ожидается 409 conflict) +OK история: ровно одно сообщение с вложением: 200 (ожидается 200 ) +OK после оборванной загрузки API работает: 201 (ожидается 201 ) +OK ошибка в общем формате с request_id: 415 unsupported_media_type (ожидается 415 ) +PASS: граничные случаи вложений соответствуют контракту diff --git a/evidence/bootstrap-idempotent.txt b/evidence/bootstrap-idempotent.txt new file mode 100644 index 00000000..0430cdfa --- /dev/null +++ b/evidence/bootstrap-idempotent.txt @@ -0,0 +1,30 @@ +## s3-init run 1 +keep bucket messenger-attachments +PASS api credentials: PUT object +PASS api credentials: GET object +PASS anonymous GET object -> 403 +PASS anonymous LIST bucket -> 403 +PASS anonymous PUT object -> 403 +PASS api credentials: CreateBucket denied +PASS api credentials: DeleteBucket denied +PASS api credentials: DELETE object +PASS api credentials: ListBuckets shows no foreign buckets +PASS: bucket bootstrap complete (idempotent, private, least privilege) +## s3-init run 2 +keep bucket messenger-attachments +PASS api credentials: PUT object +PASS api credentials: GET object +PASS anonymous GET object -> 403 +PASS anonymous LIST bucket -> 403 +PASS anonymous PUT object -> 403 +PASS api credentials: CreateBucket denied +PASS api credentials: DeleteBucket denied +PASS api credentials: DELETE object +PASS api credentials: ListBuckets shows no foreign buckets +PASS: bucket bootstrap complete (idempotent, private, least privilege) +## migrate rerun +SKIP 001_initial_schema.sql +SKIP 002_indexes.sql +SKIP 003_idempotency_keys.sql +SKIP 004_attachments.sql +PASS: migrations applied diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..5e20948e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,24 @@ +## run 1 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 3.972} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +messenger-lab4-api-2 bb5edb449d7c Up 41 seconds (healthy) +messenger-lab4-api-1 f00b790a44e9 Exited (0) 12 seconds ago +messenger-lab3-api-6 82d4e8380bcc Exited (0) 6 days ago +messenger-lab3-api-5 9478eeefc0a7 Exited (0) 6 days ago +mtusi2026lab2-api-1 01469cb1e76e Exited (137) 2 weeks ago +started api container f00b790a44e9 +running api replicas: 2 +## run 2 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 4.208} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +messenger-lab4-api-2 bb5edb449d7c Up About a minute (healthy) +messenger-lab4-api-1 f00b790a44e9 Exited (0) 11 seconds ago +messenger-lab3-api-6 82d4e8380bcc Exited (0) 6 days ago +messenger-lab3-api-5 9478eeefc0a7 Exited (0) 6 days ago +mtusi2026lab2-api-1 01469cb1e76e Exited (137) 2 weeks ago +started api container f00b790a44e9 +running api replicas: 2 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..f3b27960 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,22 @@ +## before +messenger-lab4-api-1 f00b790a44e9 started=2026-10-30T11:44:20.074794065Z status=running health=healthy +messenger-lab4-api-2 bb5edb449d7c started=2026-10-30T11:44:19.748733251Z status=running health=healthy +messenger-lab4-db-1 331396b7fb10 started=2026-10-30T11:09:07.431846589Z status=running health=healthy +messenger-lab4-lb-1 ffe0343f6dc2 started=2026-10-30T11:23:41.657935704Z status=running health=healthy +messenger-lab4-s3-1 16bae2ec7298 started=2026-10-30T11:45:54.317998359Z status=running health=healthy +messenger-lab4-valkey-1 b7f2d1a478a8 started=2026-10-30T11:09:07.702076066Z status=running health=healthy +volume messenger-lab4_postgres_data +volume messenger-lab4_s3_data +## test +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +## after +messenger-lab4-api-1 f00b790a44e9 started=2026-10-30T11:47:37.860335522Z status=running health=healthy +messenger-lab4-api-2 bb5edb449d7c started=2026-10-30T11:47:37.593545231Z status=running health=healthy +messenger-lab4-db-1 331396b7fb10 started=2026-10-30T11:47:11.170703341Z status=running health=healthy +messenger-lab4-lb-1 ffe0343f6dc2 started=2026-10-30T11:47:44.439408787Z status=running health=starting +messenger-lab4-s3-1 16bae2ec7298 started=2026-10-30T11:47:11.637306219Z status=running health=healthy +messenger-lab4-valkey-1 b7f2d1a478a8 started=2026-10-30T11:47:11.434151225Z status=running health=healthy +volume messenger-lab4_postgres_data +volume messenger-lab4_s3_data diff --git a/evidence/s3-failures.txt b/evidence/s3-failures.txt new file mode 100644 index 00000000..1a78f061 --- /dev/null +++ b/evidence/s3-failures.txt @@ -0,0 +1,34 @@ + Container messenger-lab4-tester-run-cabae88c67e1 Creating + Container messenger-lab4-tester-run-cabae88c67e1 Created +OK object_key есть в PostgreSQL и не совпадает с filename/ID +OK API не раскрывает object_key +OK анонимный GET /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный HEAD /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный GET /messenger-attachments?list-type=2 напрямую в S3 -> 403 (403) +OK анонимный GET /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный PUT /messenger-attachments/ напрямую в S3 -> 403 (403) +OK анонимный DELETE /messenger-attachments/ напрямую в S3 -> 403 (403) +OK объект не изменён анонимными PUT/DELETE +OK S3 не опубликован на хосте (docker port пуст) '' +OK LB не проксирует путь S3 (404) +OK SeaweedFS master UI/API :9333 недоступен из сети стенда +OK SeaweedFS filer UI/API :8888 недоступен из сети стенда +OK SeaweedFS volume :8080 недоступен из сети стенда +OK SeaweedFS master gRPC :19333 недоступен из сети стенда +OK SeaweedFS filer gRPC :18888 недоступен из сети стенда +OK SeaweedFS admin :23646 недоступен из сети стенда +-- s3 остановлен +OK /health/ready при отказе S3 -> 200 (S3 не критичен для чата) (200) +OK сообщение со ссылкой на готовое вложение -> 201 (201) +OK metadata из PostgreSQL -> 200 (200) +OK upload -> 503 unavailable (503 unavailable, 6.13 s) +OK upload отказал быстрее timeout LB (< 8 s) (6.13 s) +OK download -> 503 unavailable (503 unavailable, 6.01 s) +OK download отказал быстрее timeout LB (< 8 s) (6.01 s) +OK пользовательская ошибка при отказе S3 остаётся 415 (415) + upload_intents после неудачных upload: 2 +-- s3 запущен +OK s3 снова healthy +OK после восстановления S3 upload -> 201 (201) +OK старый файл скачивается побайтно (200) +PASS: bucket приватен вне API, отказ S3 даёт 503 только для вложений diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/upload-compensation.txt b/evidence/upload-compensation.txt new file mode 100644 index 00000000..4efbb8b1 --- /dev/null +++ b/evidence/upload-compensation.txt @@ -0,0 +1,11 @@ +OK db_error_after_s3_put: 503 (503) +OK db_error_after_s3_put: компенсация убрала intent +OK crash_after_s3_put: клиент не получил 201 (RemoteDisconnected) +OK crash_after_s3_put: остались intent и объект +OK janitor до grace ничего не удаляет +-- ждём grace 21 s +OK janitor после grace удалил intent от crash (cleaned=1) +OK janitor удалил объект от crash +OK свежий intent не тронут +OK готовое вложение цело +PASS: компенсация и janitor не оставляют вечных объектов diff --git a/evidence/upload-failover.txt b/evidence/upload-failover.txt new file mode 100644 index 00000000..39ec4fcc --- /dev/null +++ b/evidence/upload-failover.txt @@ -0,0 +1,3 @@ +OK upload через bb5edb449d7c -> bb5edb449d7c остановлена -> download через ['f00b790a44e9']: SHA-256 совпал +OK upload через f00b790a44e9 -> f00b790a44e9 остановлена -> download через ['bb5edb449d7c']: SHA-256 совпал +PASS: файл доступен после остановки любой обслужившей реплики diff --git a/evidence/upload-memory-c12.json b/evidence/upload-memory-c12.json new file mode 100644 index 00000000..e5c88e5a --- /dev/null +++ b/evidence/upload-memory-c12.json @@ -0,0 +1,74 @@ +{ + "concurrency": 12, + "file_bytes": 9962537, + "statuses": { + "201": 12 + }, + "per_instance": { + "b4841ddecd64": 6, + "dd76ecc96935": 6 + }, + "wall_seconds": 5.18, + "latency_p50_s": 4.98, + "latency_max_s": 5.17, + "sha256_ok": true, + "memory_mib_before": { + "dd76ecc96935": { + "VmHWM": 169.2, + "VmRSS": 105.2, + "peak": 148.3, + "current": 89.0, + "anon": 87.5 + }, + "b4841ddecd64": { + "VmHWM": 169.4, + "VmRSS": 105.6, + "peak": 148.7, + "current": 85.2, + "anon": 83.2 + } + }, + "memory_mib_peak_during": { + "dd76ecc96935": { + "VmRSS_peak": 118.1, + "anon_peak": 108.3, + "samples": 57 + }, + "b4841ddecd64": { + "VmRSS_peak": 119.3, + "anon_peak": 109.5, + "samples": 57 + } + }, + "memory_mib_after": { + "dd76ecc96935": { + "VmHWM": 169.2, + "VmRSS": 118.1, + "peak": 167.7, + "current": 98.5, + "anon": 95.9 + }, + "b4841ddecd64": { + "VmHWM": 169.4, + "VmRSS": 119.3, + "peak": 174.5, + "current": 99.9, + "anon": 97.2 + } + }, + "rss_peak_increase_mib": { + "dd76ecc96935": 12.9, + "b4841ddecd64": 13.7 + }, + "anon_peak_increase_mib": { + "dd76ecc96935": 20.8, + "b4841ddecd64": 26.3 + }, + "s3_server_mib": { + "VmRSS_before": 203.5, + "VmRSS_peak": 546.1, + "anon_peak": 495.1, + "samples": 47 + }, + "naive_in_memory_estimate_mib": 114.0 +} diff --git a/evidence/upload-memory-c20.json b/evidence/upload-memory-c20.json new file mode 100644 index 00000000..ceb95f92 --- /dev/null +++ b/evidence/upload-memory-c20.json @@ -0,0 +1,74 @@ +{ + "concurrency": 20, + "file_bytes": 9962537, + "statuses": { + "201": 20 + }, + "per_instance": { + "dd76ecc96935": 10, + "b4841ddecd64": 10 + }, + "wall_seconds": 9.89, + "latency_p50_s": 8.58, + "latency_max_s": 9.88, + "sha256_ok": true, + "memory_mib_before": { + "dd76ecc96935": { + "VmHWM": 169.4, + "VmRSS": 105.6, + "peak": 148.5, + "current": 85.6, + "anon": 83.1 + }, + "b4841ddecd64": { + "VmHWM": 169.3, + "VmRSS": 105.5, + "peak": 148.8, + "current": 85.8, + "anon": 83.2 + } + }, + "memory_mib_peak_during": { + "dd76ecc96935": { + "VmRSS_peak": 122.1, + "anon_peak": 111.0, + "samples": 98 + }, + "b4841ddecd64": { + "VmRSS_peak": 123.2, + "anon_peak": 113.8, + "samples": 97 + } + }, + "memory_mib_after": { + "dd76ecc96935": { + "VmHWM": 169.4, + "VmRSS": 116.7, + "peak": 195.2, + "current": 97.8, + "anon": 94.5 + }, + "b4841ddecd64": { + "VmHWM": 169.3, + "VmRSS": 123.2, + "peak": 196.0, + "current": 104.3, + "anon": 101.4 + } + }, + "rss_peak_increase_mib": { + "dd76ecc96935": 16.5, + "b4841ddecd64": 17.7 + }, + "anon_peak_increase_mib": { + "dd76ecc96935": 27.9, + "b4841ddecd64": 30.6 + }, + "s3_server_mib": { + "VmRSS_before": 205.3, + "VmRSS_peak": 778.8, + "anon_peak": 698.4, + "samples": 80 + }, + "naive_in_memory_estimate_mib": 190.0 +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 971d1e87..40239779 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -81,6 +81,12 @@ backend api http-request disable-l7-retry if METH_POST !is_message_post http-request disable-l7-retry if METH_POST !has_idem_key http-request disable-l7-retry if METH_DELETE + # Lab4: 503 вложения означает отказ общего S3 — на другой реплике тот же S3, повтор + # только утроил бы ожидание клиента (3 × S3_OPERATION_TIMEOUT_SECONDS). conn-failure остаётся. + http-request disable-l7-retry if { path_beg /api/v1/attachments/ } + # Upload: после приёма тела API проверяет PNG/JPEG и делает S3 PUT с дедлайном + # 6 s + size / 1 MiB/s (<= 16 s для 10 MiB) — 10 s «timeout server» мало. + http-request set-timeout server 40s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ } # До 8 реплик (scale/rolling update); пустые слоты остаются в MAINT, пока DNS не вернёт адрес. server-template api 8 api:8080 diff --git a/infra/seaweedfs/s3.json.template b/infra/seaweedfs/s3.json.template new file mode 100644 index 00000000..afdd0046 --- /dev/null +++ b/infra/seaweedfs/s3.json.template @@ -0,0 +1,22 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { "accessKey": "@ADMIN_ACCESS_KEY@", "secretKey": "@ADMIN_SECRET_KEY@" } + ], + "actions": ["Admin", "Read", "List", "Tagging", "Write"] + }, + { + "name": "messenger-api", + "credentials": [ + { "accessKey": "@API_ACCESS_KEY@", "secretKey": "@API_SECRET_KEY@" } + ], + "actions": [ + "Read:messenger-attachments", + "Write:messenger-attachments", + "List:messenger-attachments" + ] + } + ] +} diff --git a/infra/seaweedfs/start.sh b/infra/seaweedfs/start.sh new file mode 100644 index 00000000..1f63d8e3 --- /dev/null +++ b/infra/seaweedfs/start.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# SeaweedFS all-in-one (master + volume + filer + S3 gateway) for the lab stand. +# +# - master/volume/filer and their web UIs listen on 127.0.0.1 INSIDE the +# container only (-ip.bind): their HTTP API has no authentication and must +# not be reachable from the Docker network. Only the S3 gateway (9000) +# listens on all interfaces, and it requires SigV4 credentials. +# - Identities and rights come from s3.json.template; the secret values are +# substituted at start from Docker secrets into a file readable by root +# only (never stored in Git or in the image). +set -eu +umask 077 +s() { cat "/run/secrets/$1"; } +sed -e "s|@ADMIN_ACCESS_KEY@|$(s s3_admin_access_key_id)|" \ + -e "s|@ADMIN_SECRET_KEY@|$(s s3_admin_secret_access_key)|" \ + -e "s|@API_ACCESS_KEY@|$(s s3_access_key_id)|" \ + -e "s|@API_SECRET_KEY@|$(s s3_secret_access_key)|" \ + /opt/lab/s3.json.template > /tmp/s3.json + +exec weed -logtostderr=true server \ + -dir=/data \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ + -master.volumeSizeLimitMB=256 -volume.max=0 \ + -s3 -s3.port=9000 -s3.ip.bind=0.0.0.0 -s3.config=/tmp/s3.json diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..4b9a549e --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,68 @@ +-- Lab4: attachments. PostgreSQL keeps metadata, the bytes live in a private +-- S3 bucket under a server-generated object_key. Expand-only migration: the +-- lab3 API keeps working on this schema. + +-- Text may be empty when the message has attachments; "not blank without +-- attachments" is checked by the API (one message + its links are inserted +-- in one transaction). +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank_check; + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE RESTRICT, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + -- Display name only; never used as a path or an S3 key. + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + -- Lab4 has only "ready": the row is inserted after the object is stored. + status VARCHAR(16) NOT NULL DEFAULT 'ready', + error_code VARCHAR(64), + object_key TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT attachments_content_type_check + CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_check + CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_check + CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status_check + CHECK (status IN ('ready')) +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id); + +-- A message references 0-10 attachments of its own chat; "position" keeps +-- the order of attachment_ids. One attachment may be referenced by several +-- messages of the chat. +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + position SMALLINT NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx + ON message_attachments (attachment_id); + +-- Upload intent: committed BEFORE the S3 PUT and deleted in the same +-- transaction that inserts the attachment row. An intent older than the +-- grace period therefore means "the object may exist in S3, but the request +-- never committed" (crash, lost DB, client gone) -> the janitor deletes the +-- object and then the intent. +CREATE TABLE IF NOT EXISTS upload_intents ( + object_key TEXT PRIMARY KEY, + attachment_id UUID NOT NULL, + chat_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS upload_intents_created_idx + ON upload_intents (created_at); diff --git a/requirements.txt b/requirements.txt index 9e3cb890..efc66e53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,12 @@ -fastapi==0.115.0 +fastapi==0.115.0 uvicorn[standard]==0.30.6 pydantic==2.9.2 SQLAlchemy==2.0.36 asyncpg==0.30.0 argon2-cffi==23.1.0 redis==5.0.8 +# Lab4: S3 client (async) and image header/JPEG decoding +aiobotocore==2.15.2 +botocore==1.35.36 +aiohttp==3.10.10 +Pillow==11.0.0 diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh index 42e3c0fd..d91f686a 100644 --- a/scripts/init-secrets.sh +++ b/scripts/init-secrets.sh @@ -34,3 +34,10 @@ if [ ! -f "$dir/valkey_password.txt" ]; then fi valkey_password="$(cat "$dir/valkey_password.txt")" write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0" + +# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init +# bootstrap; the API gets its own key pair restricted to one bucket. +write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)" +write_new s3_admin_secret_access_key.txt "$(random)$(random)" +write_new s3_access_key_id.txt "api$(random | cut -c1-17)" +write_new s3_secret_access_key.txt "$(random)$(random)" diff --git a/scripts/lib.sh b/scripts/lib.sh index 688836b6..0877da12 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,7 +1,7 @@ #!/bin/sh # Общие функции операторских скриптов. Используют только docker CLI и метки Compose, # поэтому работают и на хосте, и в контейнере tester (docker.sock). -PROJECT="${COMPOSE_PROJECT:-messenger-lab3}" +PROJECT="${COMPOSE_PROJECT:-messenger-lab4}" # ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные. service_ids() { diff --git a/scripts/restart.sh b/scripts/restart.sh index 3492ae6f..69df0e94 100644 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,19 +1,20 @@ #!/bin/sh -# Persistence hook: restarts the LB, every API replica and both stores -# (PostgreSQL with its volume, Valkey cache). No "down -v", no re-seed. +# Persistence hook: restarts the LB, every API replica and all stores +# (PostgreSQL and S3 with their volumes, Valkey cache). No "down -v", no re-seed. # Uses only the docker CLI and Compose labels, so it also runs in "tester". set -eu . "$(dirname "$0")/lib.sh" -api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db); valkey=$(service_ids valkey) -[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] || { echo "stack is not running" >&2; exit 2; } -echo "stopping lb + api, then db + valkey" +api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db) +valkey=$(service_ids valkey); s3=$(service_ids s3) +[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] && [ -n "$s3" ] || { echo "stack is not running" >&2; exit 2; } +echo "stopping lb + api, then db + valkey + s3" docker stop --time 15 $lb $api >/dev/null -docker stop --time 30 $db $valkey >/dev/null -echo "starting db + valkey" -docker start $db $valkey >/dev/null -wait_healthy 60 $db $valkey +docker stop --time 30 $db $valkey $s3 >/dev/null +echo "starting db + valkey + s3" +docker start $db $valkey $s3 >/dev/null +wait_healthy 80 $db $valkey $s3 echo "starting api" docker start $api >/dev/null wait_healthy 60 $api docker start $lb >/dev/null -echo "restart complete: db=$(short $db) valkey=$(short $valkey) api=$(for i in $api; do short $i; done | tr '\n' ' ')" +echo "restart complete: db=$(short $db) valkey=$(short $valkey) s3=$(short $s3) api=$(for i in $api; do short $i; done | tr '\n' ' ')" diff --git a/scripts/s3_bootstrap.py b/scripts/s3_bootstrap.py new file mode 100644 index 00000000..79276d81 --- /dev/null +++ b/scripts/s3_bootstrap.py @@ -0,0 +1,161 @@ +"""Idempotent bootstrap of the private attachments bucket (service s3-init). + +Runs with the S3 ADMIN credentials (only this one-shot container gets them), +before any API replica starts: + + 1. wait until the S3 gateway answers; + 2. create the bucket if it does not exist (a repeated run prints "keep"); + 3. self-checks, the stand does not start if one fails: + - anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private; + - API credentials can PUT/GET/DELETE under the prefix; + - API credentials can NOT create/delete buckets or touch other buckets. + +The API identity and its rights are declared in infra/seaweedfs/s3.json.template. +""" +from __future__ import annotations + +import os +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +import botocore.session +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + + +def secret(name: str) -> str: + file_name = os.getenv(f"{name}_FILE") + if file_name: + return Path(file_name).read_text(encoding="utf-8").strip() + value = os.getenv(name) + if not value: + raise RuntimeError(f"{name} or {name}_FILE is required") + return value + + +ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/") + + +def client(access_key: str, secret_key: str): + return botocore.session.get_session().create_client( + "s3", + endpoint_url=ENDPOINT, + region_name=os.getenv("S3_REGION", "us-east-1"), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 1}, + ), + ) + + +def error_code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "") + + +failures = 0 + + +def check(label: str, ok: bool) -> None: + global failures + failures += not ok + print(("PASS " if ok else "FAIL ") + label, flush=True) + + +def denied(call) -> bool: + try: + call() + except ClientError as exc: + return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"} + return False + + +def anonymous_status(method: str, path: str, data: bytes | None = None) -> int: + request = urllib.request.Request(ENDPOINT + path, data=data, method=method) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY")) + + for attempt in range(1, 61): + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if attempt == 60: + raise + print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True) + time.sleep(1) + + try: + admin.head_bucket(Bucket=BUCKET) + print(f"keep bucket {BUCKET}", flush=True) + except ClientError as exc: + if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}: + raise + try: + admin.create_bucket(Bucket=BUCKET) + print(f"create bucket {BUCKET}", flush=True) + except ClientError as create_exc: + # Two bootstrap runs at the same time: the other one won. + if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + raise + print(f"keep bucket {BUCKET}", flush=True) + + probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}" + + check("api credentials: PUT object", not denied( + lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe"))) + check("api credentials: GET object", + api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe") + + check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403) + check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403) + check("anonymous PUT object -> 403", + anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403) + + other = f"forbidden-{uuid.uuid4().hex[:12]}" + check("api credentials: CreateBucket denied", + denied(lambda: api.create_bucket(Bucket=other))) + check("api credentials: DeleteBucket denied", + denied(lambda: api.delete_bucket(Bucket=BUCKET))) + check("api credentials: DELETE object", not denied( + lambda: api.delete_object(Bucket=BUCKET, Key=probe))) + + def foreign_buckets_hidden() -> bool: + # SeaweedFS denies ListBuckets to a bucket-scoped identity; other + # servers may answer with only the allowed bucket. Both are fine. + try: + names = [b["Name"] for b in api.list_buckets().get("Buckets", [])] + except ClientError as exc: + return error_code(exc) in {"AccessDenied", "403", "Forbidden"} + return all(name == BUCKET for name in names) + + check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden()) + + if failures: + print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True) + return 1 + + print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/attachment_edge_cases.py b/student-tests/attachment_edge_cases.py new file mode 100644 index 00000000..4e2344b6 --- /dev/null +++ b/student-tests/attachment_edge_cases.py @@ -0,0 +1,327 @@ +"""Негативные и граничные сценарии вложений lab4 (только stdlib). + +Проверяет, что пользовательская ошибка никогда не становится 500: + - размер: ровно 10 MiB -> 201; 10 MiB + 1 по Content-Length и chunked -> 413; + - пиксели: ровно 40 млн -> 201; 50 млн (маленький файл, «бомба») -> 413; + - повреждённые PNG/JPEG (обрезка, CRC, мусор в IDAT), чужие форматы -> 415; + - подмена MIME: PNG как image/jpeg, JPEG как image/png, text/plain, multipart -> 415; + - опасные filename (/, \\, NUL, управляющие, bidi, пустой, 256 символов) -> 400; + - чужой пользователь: upload в чужой чат, metadata, content -> 404; + - message: вложение другого чата / неизвестное -> 404, дубли / 11 ID -> 400, + text="" с вложением -> 201, повтор Idempotency-Key -> то же сообщение; + - оборванная загрузка не ломает реплику; variant без варианта -> 404. + +Запуск: docker compose --profile tools run --rm tester python3 student-tests/attachment_edge_cases.py +или с хоста: python student-tests/attachment_edge_cases.py --base-url http://localhost:8080 +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import http.client +import json +import os +import socket +import struct +import sys +import urllib.parse +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE_PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes() +SAMPLE_JPG = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes() +MIB10 = 10 * 1024 * 1024 + + +# --- синтетические изображения --------------------------------------------- + +def png_chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + + +def png(width: int, height: int, bit_depth: int = 8, color_type: int = 0, idat: bytes | None = None, + extra: bytes = b'') -> bytes: + if idat is None: + row_bytes = (width * bit_depth * {0: 1, 2: 3}[color_type] + 7) // 8 + raw = (b'\x00' + b'\x00' * row_bytes) * height + idat = zlib.compress(raw, 9) + ihdr = struct.pack('>IIBBBBB', width, height, bit_depth, color_type, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', ihdr) + extra + + png_chunk(b'IDAT', idat) + png_chunk(b'IEND', b'')) + + +def png_exactly(size: int) -> bytes: + """Валидный PNG ровно size байт: добивается приватным ancillary-чанком.""" + base = png(8, 8) + pad = size - len(base) - 12 + return png(8, 8, extra=png_chunk(b'zpAD', b'\x00' * pad)) + + +def pixels_png(width: int, height: int) -> bytes: + """1-битный grayscale: 40 млн пикселей занимают ~5 MB raw, ~10 KB zlib.""" + row = b'\x00' + b'\x00' * ((width + 7) // 8) + compressor = zlib.compressobj(9) + idat = b''.join(compressor.compress(row) for _ in range(height)) + compressor.flush() + return png(width, height, bit_depth=1, idat=idat) + + +def corrupt_crc(data: bytes) -> bytes: + i = data.index(b'IDAT') + 6 + return data[:i] + bytes([data[i] ^ 0xFF]) + data[i + 1:] + + +def corrupt_idat(data: bytes) -> bytes: + """Мусор в zlib-потоке с ПРАВИЛЬНЫМ CRC: ловится только распаковкой.""" + return png(320, 200, color_type=2, idat=b'\x78\x9c' + os.urandom(2000)) + + +# --- HTTP ------------------------------------------------------------------ + +class Api: + def __init__(self, base_url: str): + parts = urllib.parse.urlsplit(base_url) + self.host = parts.hostname + self.port = parts.port or 80 + + def conn(self) -> http.client.HTTPConnection: + return http.client.HTTPConnection(self.host, self.port, timeout=60) + + def call(self, method, path, body=None, headers=None, raw=None, chunks=None): + h = dict(headers or {}) + if body is not None: + raw = json.dumps(body).encode() + h['Content-Type'] = 'application/json' + c = self.conn() + try: + if chunks is not None: + c.request(method, '/api/v1' + path, body=chunks, headers=h, encode_chunked=True) + else: + c.request(method, '/api/v1' + path, body=raw, headers=h) + r = c.getresponse() + data = r.read() + except (BrokenPipeError, ConnectionResetError): + # Сервер ответил 413 и закрыл соединение, не дочитав тело. + r = c.getresponse() + data = r.read() + finally: + c.close() + ctype = r.getheader('Content-Type', '') + parsed = json.loads(data) if data and ctype.startswith('application/json') else None + return r.status, parsed, data, r + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + args = p.parse_args() + api = Api(args.base_url) + failed = 0 + + def code(resp): + return resp.get('error', {}).get('code') if isinstance(resp, dict) else None + + def check(label, status, resp, want_status, want_code=None, extra_ok=True): + nonlocal failed + ok = status == want_status and (want_code is None or code(resp) == want_code) and extra_ok + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label}: {status} {code(resp) or ''} " + f"(ожидается {want_status} {want_code or ''})", flush=True) + + def new_user(): + name = 'att' + uuid.uuid4().hex[:12] + password = 'attachment-edge-password' + s, user, _, _ = api.call('POST', '/users', {'username': name, 'display_name': 'Вложения', 'password': password}) + assert s == 201, f'register: {s}' + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + s, session, _, _ = api.call('POST', '/auth/sessions', headers={'Authorization': basic}) + assert s == 201, f'login: {s}' + return user, {'Authorization': 'Bearer ' + session['token']} + + def new_chat(auth, members, title='attachments'): + s, chat, _, _ = api.call('POST', '/chats', {'title': title, 'member_ids': members}, headers=auth) + assert s == 201, f'chat: {s}' + return chat + + def upload(auth, chat_id, data, content_type='image/png', filename='photo.png', **kw): + query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='') + h = dict(auth) + if content_type is not None: + h['Content-Type'] = content_type + return api.call('POST', f'/chats/{chat_id}/attachments{query}', headers=h, raw=data, **kw) + + def message(auth, chat_id, text, ids=None, key=None): + body = {'text': text} + if ids is not None: + body['attachment_ids'] = ids + h = dict(auth, **{'Idempotency-Key': key or uuid.uuid4().hex}) + return api.call('POST', f'/chats/{chat_id}/messages', body, headers=h) + + alice, a_auth = new_user() + bob, b_auth = new_user() + eve, e_auth = new_user() + chat = new_chat(a_auth, [alice['id'], bob['id']]) + other_chat = new_chat(a_auth, [alice['id']], 'other') + eve_chat = new_chat(e_auth, [eve['id']], 'eve') + + # --- успешные границы --------------------------------------------------- + s, att, _, r = upload(a_auth, chat['id'], SAMPLE_JPG, 'image/jpeg', 'фото 😀 v1.0.jpg') + check('JPEG fixture -> 201 ready', s, att, 201, extra_ok=s == 201 and att['status'] == 'ready' + and att['sha256'] == hashlib.sha256(SAMPLE_JPG).hexdigest() and att['size_bytes'] == len(SAMPLE_JPG) + and att['filename'] == 'фото 😀 v1.0.jpg' and att['variants'] == [] and att['error_code'] is None + and 'object_key' not in att) + jpg_id = att['id'] + s, _, data, r = api.call('GET', f'/attachments/{jpg_id}/content', headers=b_auth) + check('участник скачивает JPEG побайтно, без redirect', s, None, 200, + extra_ok=data == SAMPLE_JPG and r.getheader('Content-Type') == 'image/jpeg') + + exact = png_exactly(MIB10) + s, att, _, _ = upload(a_auth, chat['id'], exact) + check('PNG ровно 10 MiB -> 201', s, att, 201, extra_ok=s == 201 and att['size_bytes'] == MIB10) + if s == 201: + s, _, data, _ = api.call('GET', f"/attachments/{att['id']}/content", headers=a_auth) + check('10 MiB скачан побайтно', s, None, 200, extra_ok=hashlib.sha256(data).digest() == hashlib.sha256(exact).digest()) + + s, att, _, _ = upload(a_auth, chat['id'], pixels_png(8000, 5000)) + check('PNG ровно 40 млн пикселей -> 201', s, att, 201) + + # --- размер ------------------------------------------------------------- + s, r, _, _ = upload(a_auth, chat['id'], png_exactly(MIB10 + 1)) + check('10 MiB + 1 байт (Content-Length) -> 413', s, r, 413, 'payload_too_large') + + def big_chunks(): + block = png_exactly(MIB10)[:1024 * 1024] + for _ in range(11): + yield block + + s, r, _, _ = upload(a_auth, chat['id'], None, chunks=big_chunks()) + check('11 MiB chunked без Content-Length -> 413', s, r, 413, 'payload_too_large') + + s, r, _, _ = upload(a_auth, chat['id'], pixels_png(10000, 5000)) + check('PNG-«бомба» 50 млн пикселей (~10 KB файл) -> 413', s, r, 413, 'payload_too_large') + s, r, _, _ = upload(a_auth, chat['id'], png(0x7FFFFFFF, 0x7FFFFFFF, idat=b'x')) + check('PNG 2^31-1 x 2^31-1 в IHDR -> 413', s, r, 413, 'payload_too_large') + + s, r, _, _ = upload(a_auth, chat['id'], b'') + check('пустое тело -> 400', s, r, 400, 'invalid_request') + + # --- повреждённые и чужие форматы -------------------------------------- + for label, data, ctype in [ + ('PNG обрезан пополам', SAMPLE_PNG[:len(SAMPLE_PNG) // 2], 'image/png'), + ('PNG с неверным CRC', corrupt_crc(SAMPLE_PNG), 'image/png'), + ('PNG с мусором в zlib при верном CRC', corrupt_idat(SAMPLE_PNG), 'image/png'), + ('PNG без IEND', SAMPLE_PNG[:-12], 'image/png'), + ('JPEG обрезан', SAMPLE_JPG[:len(SAMPLE_JPG) // 2], 'image/jpeg'), + ('JPEG только сигнатура', b'\xff\xd8\xff\xe0' + b'\x00' * 100, 'image/jpeg'), + ('GIF как image/png', b'GIF89a' + b'\x00' * 100, 'image/png'), + ('случайные байты как image/png', os.urandom(4096), 'image/png'), + ('HTML как image/jpeg', b'', 'image/jpeg'), + ]: + s, r, _, _ = upload(a_auth, chat['id'], data, ctype) + check(label + ' -> 415', s, r, 415, 'unsupported_media_type') + + # --- подмена MIME ------------------------------------------------------- + for label, data, ctype in [ + ('PNG под Content-Type image/jpeg', SAMPLE_PNG, 'image/jpeg'), + ('JPEG под Content-Type image/png', SAMPLE_JPG, 'image/png'), + ('PNG под text/plain', SAMPLE_PNG, 'text/plain'), + ('PNG под image/gif', SAMPLE_PNG, 'image/gif'), + ('multipart/form-data', b'--x\r\n\r\n' + SAMPLE_PNG + b'\r\n--x--', 'multipart/form-data; boundary=x'), + ('без Content-Type', SAMPLE_PNG, None), + ]: + s, r, _, _ = upload(a_auth, chat['id'], data, ctype) + check(label + ' -> 415', s, r, 415, 'unsupported_media_type') + + # --- filename ----------------------------------------------------------- + for label, name in [ + ('../../etc/passwd', '../../etc/passwd'), + ('C:\\boot.png', 'C:\\boot.png'), + ('NUL в имени', 'a\x00.png'), + ('перевод строки', 'a\nb.png'), + ('ESC-последовательность', '\x1b[31mred.png'), + ('bidi override (gnp.exe)', 'photo\u202egnp.exe'), + ('только пробелы', ' '), + ('..', '..'), + ('256 символов', 'a' * 252 + '.png'), + ('пустой filename', ''), + ('нет filename', None), + ]: + s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG, filename=name) + check(f'filename {label} -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG, filename='a' * 251 + '.png') + check('filename ровно 255 символов -> 201', s, r, 201) + + # --- доступ ------------------------------------------------------------- + s, r, _, _ = upload({}, chat['id'], SAMPLE_PNG) + check('upload без токена -> 401', s, r, 401, 'unauthorized') + s, r, _, _ = upload(e_auth, chat['id'], SAMPLE_PNG) + check('посторонний upload в чужой чат -> 404', s, r, 404, 'not_found') + s, r, _, _ = upload(a_auth, str(uuid.uuid4()), SAMPLE_PNG) + check('upload в несуществующий чат -> 404', s, r, 404, 'not_found') + for suffix in ('', '/content'): + s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}{suffix}', headers=e_auth) + check(f'посторонний GET /attachments/id{suffix} -> 404', s, r, 404, 'not_found') + s, r, _, _ = api.call('GET', f'/attachments/{uuid.uuid4()}{suffix}', headers=a_auth) + check(f'неизвестный attachment{suffix} -> 404', s, r, 404, 'not_found') + s, r, _, _ = api.call('GET', '/attachments/not-a-uuid', headers=a_auth) + check('attachment_id не UUID -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}/content?variant=thumbnail', headers=a_auth) + check('ready без варианта thumbnail -> 404', s, r, 404, 'not_found') + s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}/content?variant=../../x', headers=a_auth) + check('variant неизвестного имени -> 400', s, r, 400, 'invalid_request') + + # --- привязка к сообщению ----------------------------------------------- + s, other_att, _, _ = upload(a_auth, other_chat['id'], SAMPLE_PNG) + s, eve_att, _, _ = upload(e_auth, eve_chat['id'], SAMPLE_PNG) + s, r, _, _ = message(a_auth, chat['id'], 'x', [other_att['id']]) + check('вложение другого чата (своего) -> 404', s, r, 404, 'not_found') + s, r, _, _ = message(a_auth, chat['id'], 'x', [eve_att['id']]) + check('вложение чужого чата -> 404', s, r, 404, 'not_found') + s, r, _, _ = message(a_auth, chat['id'], 'x', [jpg_id, str(uuid.uuid4())]) + check('одно из ID неизвестно -> 404, ничего не создано', s, r, 404, 'not_found') + s, r, _, _ = message(a_auth, chat['id'], 'x', [jpg_id, jpg_id]) + check('дубли attachment_ids -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = message(a_auth, chat['id'], 'x', [str(uuid.uuid4()) for _ in range(11)]) + check('11 attachment_ids -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = message(a_auth, chat['id'], 'x', ['not-a-uuid']) + check('attachment_id не UUID -> 400', s, r, 400, 'invalid_request') + s, r, _, _ = message(a_auth, chat['id'], ' ', []) + check('пробельный text без вложений -> 400', s, r, 400, 'invalid_request') + + key = uuid.uuid4().hex + s, m1, _, _ = message(b_auth, chat['id'], '', [jpg_id], key) + check('Bob: text="" + вложение Alice из общего чата -> 201', s, m1, 201, + extra_ok=s == 201 and m1['attachment_ids'] == [jpg_id] and m1['text'] == '') + s, m2, _, _ = message(b_auth, chat['id'], '', [jpg_id], key) + check('повтор Idempotency-Key -> то же сообщение', s, m2, 201, extra_ok=s == 201 and m2['id'] == m1['id']) + s, r, _, _ = message(b_auth, chat['id'], 'другой текст', [], key) + check('тот же ключ, другое тело -> 409', s, r, 409, 'conflict') + s, page, _, _ = api.call('GET', f"/chats/{chat['id']}/messages?limit=100", headers=a_auth) + linked = [m for m in page['items'] if m['attachment_ids']] + check('история: ровно одно сообщение с вложением', s, page, 200, + extra_ok=len(linked) == 1 and linked[0]['attachment_ids'] == [jpg_id]) + + # --- оборванная загрузка ------------------------------------------------ + sock = socket.create_connection((api.host, api.port), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\n" + f"Host: {api.host}\r\nAuthorization: {a_auth['Authorization']}\r\n" + f"Content-Type: image/png\r\nContent-Length: {len(SAMPLE_PNG)}\r\n\r\n") + sock.sendall(head.encode() + SAMPLE_PNG[:len(SAMPLE_PNG) // 3]) + sock.close() + s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG) + check('после оборванной загрузки API работает', s, r, 201) + + # --- формат ошибок ------------------------------------------------------ + s, r, _, resp = upload(a_auth, chat['id'], b'not an image') + check('ошибка в общем формате с request_id', s, r, 415, extra_ok=bool(r) + and r['error']['request_id'] == resp.getheader('X-Request-Id') and bool(resp.getheader('X-Instance-Id'))) + + print('PASS: граничные случаи вложений соответствуют контракту' if not failed + else f'FAIL: {failed} проверок не прошло') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1bafc90095edef09ee47698db1956206cb292274 GIT binary patch literal 4236 zcmbVOe^8ru8GrKwNYbt)fwZ+Pzd=g2KN!poa&V<#lr7gIKwjI<+g;bu7UibbuEip_ zDJ-n4-cdV2sp~RR2jPu96)W4Scb-JH&RXZq657K4FifirhX*RSStov;=li}1Y&}m+ zuUy_d-|zE2KR%z&^DN(3{+*QS-u1{XNvD$}9sWtn(^9pRWH1_yh9vw+N=iyjS!+tc zXZ@Nr=C#)KHk)<5)ta_p>&CS7&33DGQ_iN%85vnwS!o+{^K&!vw`OK#k`o=SrX;7V zGnv+9rd!i9|L4#0&n2r#dP`Sj(B(>ctIlB6EsslCh%-sYA4&b_^adlMG?^`Ha9}i5 z((4QcJ>o|KIQ=y2lF^!E%ejANa$3WIl-#4~C7oviraafkn7#3x`TWu+p6W7NHf-GV ziOmI{EZkP)c;HjpA1o_(RzFm;tM=hX++O+9d-m>o^fOHd4;^m)?B`mJ9dG^OiMB7b zKkMs$?qtvNr(XC*U;j71HE{OZ=fC^P;81W_3H|8ek4Im9?e$AP3IAgJjW;j9^~)>2 zy*3eg(#CoF6HmQk*^vK0v|xc!?Rj1V?|NN*0+b{9J9ylH3oeb-lYebj>(;{ye?zl`WWO`}Daink2eX{iX^qz?0aadkF8?pFu;E5ji7=_c1*i=ibzu4V|I~*H1 zpgY$sJ{^vfd0JGH9Ug>loD(6a9&~IGzBBXlf)`rh@ATBqZeIOYaktBwQBmtfH9F+- zm+FUIBiF`Ti#;vv!O_FFT3rECuYY7>tWt?qhA8VCt1!2DT7G>s*6p(Q`ty`m2;LhM^S)m%ZyRDb{Ee{`a2I{|B>)97O;-6T^5>|=gM)xSI zVL5x>?HMN`sq{`=Jm!qXPtW@!)Q`-31wjA?p}o`PFTXKA?6Uc7Luh$DR|vg12XFD)`H4%K zZ+5qk4fGDwdaT!{FTMO`Dpf-9-q)Jli@F8sI~)DB;J(|l8QdU(-O-H7ro^jzP-cd) zUhb?69RTo<3RVMt|3NDe!{E_@&xdbR4G zxNuasuOF^Lol%dT7`K{AaN+nu2HHTiM(+q3WJ018#)61G2jnWb?6FRMfruLdi+_b& zI1C?Z%3%--Pd;x9_Iwy3QZD_udEv_-Ib$+fqzOQWH&V%Vj+&Vk(=@>N%mhG2j2UEH zNT9Q)u|5^cN$o2Q=KxZ51lTLJcH-k7t|_Y~yJtpu%#jn>Q3oeV1q0WW)$k678P)Q;z zlTI7BH)5@Lu_1mhBlktWXo zb{a(#mPQ7vr9az!wh0rNBc`&iT2EH!WYlp8OTuJaC&O@vR)+&7gga2%Xqsin5ie#l zDhh;m#9J1>g{CRvn(*o<-d4ah(V$6e0z6*hvmsvX{-PNt4cqGRF7Nm=5n7b)oM1Y+ z2%n&3l@KipOQ{pg7gEEI*hTKDyO`W@8OBtnK^`0>nG#InO<|=BJFz@XT_peAfy#B({k~Vm}!w(<*A?aL|nD2s;s6f21&6<7c|d{u-53{|i$NR{`ojaJ%JvG2n_4OsSzOPf{gyNtnhqMO3+%nLS!2#( zl$O(QB0pF literal 0 HcmV?d00001 diff --git a/student-tests/s3_failures.py b/student-tests/s3_failures.py new file mode 100644 index 00000000..c473dcb4 --- /dev/null +++ b/student-tests/s3_failures.py @@ -0,0 +1,185 @@ +"""Приватность bucket вне API и недоступность S3 (stdlib + docker CLI). + +1. Bucket приватен вне API: реальный object_key берётся из PostgreSQL + (docker exec db psql), затем анонимные GET/HEAD/LIST/PUT напрямую в S3 + из сети backend -> 403; S3-порт не опубликован на хосте; LB не + проксирует пути S3. +2. Отказ S3 (docker stop s3): /health/ready 200, сообщения 201, metadata 200, + upload/download -> быстрый 503 в общем формате (не 500, не зависание). + После docker start s3: upload 201, старый файл скачивается побайтно. + +Нужен docker.sock: запускайте только в tester на учебном стенде: + docker compose --profile tools run --rm tester python3 student-tests/s3_failures.py +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/') +S3 = os.environ.get('S3_ENDPOINT', 'http://s3:9000').rstrip('/') +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') +BUCKET = 'messenger-attachments' +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + +failed = 0 + + +def check(label, ok, detail=''): + global failed + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label} {detail}".rstrip(), flush=True) + + +def docker(*args, check_rc=True) -> str: + return subprocess.run(['docker', *args], check=check_rc, capture_output=True, text=True).stdout.strip() + + +def container(service: str) -> str: + ids = docker('ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', f'label=com.docker.compose.service={service}').split() + assert ids, f'no container for {service}' + return ids[0] + + +def http(method, url, data=None, headers=None, timeout=30): + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + started = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read(), time.monotonic() - started + except urllib.error.HTTPError as e: + return e.code, e.read(), time.monotonic() - started + + +def api(method, path, body=None, auth=None, raw=None, ctype=None, extra=None): + h = dict(extra or {}) + if auth: + h['Authorization'] = auth + if body is not None: + raw = json.dumps(body).encode() + ctype = 'application/json' + if ctype: + h['Content-Type'] = ctype + status, data, elapsed = http(method, BASE + '/api/v1' + path, raw, h) + try: + parsed = json.loads(data) if data else None + except ValueError: + parsed = None + return status, parsed, data, elapsed + + +def err(parsed): + return parsed.get('error', {}).get('code') if isinstance(parsed, dict) else None + + +def wait_healthy(cid, seconds=90): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy': + return True + time.sleep(1) + return False + + +def main() -> int: + name = 's3f' + uuid.uuid4().hex[:12] + password = 's3-failure-password' + s, user, _, _ = api('POST', '/users', {'username': name, 'display_name': 'S3', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + s, session, _, _ = api('POST', '/auth/sessions', auth=basic) + auth = 'Bearer ' + session['token'] + s, chat, _, _ = api('POST', '/chats', {'title': 's3', 'member_ids': [user['id']]}, auth=auth) + s, att, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=a.png", auth=auth, raw=SAMPLE, ctype='image/png') + assert s == 201, s + + # --- 1. приватный bucket вне API --------------------------------------- + db = container('db') + key = docker('exec', db, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', + f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + check('object_key есть в PostgreSQL и не совпадает с filename/ID', + bool(key) and 'a.png' not in key and att['id'] not in key) + check('API не раскрывает object_key', key not in json.dumps(att)) + for method, path, data in [ + ('GET', f'/{BUCKET}/{key}', None), + ('HEAD', f'/{BUCKET}/{key}', None), + ('GET', f'/{BUCKET}?list-type=2', None), + ('GET', f'/{BUCKET}/', None), + ('PUT', f'/{BUCKET}/{key}', b'overwrite'), + ('DELETE', f'/{BUCKET}/{key}', None), + ]: + s, _, _ = http(method, S3 + path, data) + check(f'анонимный {method} {path.replace(key, "")} напрямую в S3 -> 403', s == 403, f'({s})') + s, _, data, _ = api('GET', f"/attachments/{att['id']}/content", auth=auth) + check('объект не изменён анонимными PUT/DELETE', hashlib.sha256(data).digest() == hashlib.sha256(SAMPLE).digest()) + s3 = container('s3') + ports = docker('port', s3, check_rc=False) + check('S3 не опубликован на хосте (docker port пуст)', ports == '', repr(ports)) + s, _, _ = http('GET', BASE + f'/{BUCKET}/{key}') + check('LB не проксирует путь S3', s == 404, f'({s})') + host = urllib.parse.urlsplit(S3).hostname + for port, role in [(9333, 'master UI/API'), (8888, 'filer UI/API'), (8080, 'volume'), + (19333, 'master gRPC'), (18888, 'filer gRPC'), (23646, 'admin')]: + try: + socket.create_connection((host, port), timeout=2).close() + reachable = True + except OSError: + reachable = False + check(f'SeaweedFS {role} :{port} недоступен из сети стенда', not reachable) + + # --- 2. отказ S3 ------------------------------------------------------- + docker('stop', '--time', '10', s3) + print('-- s3 остановлен', flush=True) + try: + s, _, _ = http('GET', BASE + '/health/ready') + check('/health/ready при отказе S3 -> 200 (S3 не критичен для чата)', s == 200, f'({s})') + s, m, _, _ = api('POST', f"/chats/{chat['id']}/messages", {'text': 'без S3', 'attachment_ids': [att['id']]}, + auth=auth, extra={'Idempotency-Key': uuid.uuid4().hex}) + check('сообщение со ссылкой на готовое вложение -> 201', s == 201, f'({s})') + s, _, _, _ = api('GET', f"/attachments/{att['id']}", auth=auth) + check('metadata из PostgreSQL -> 200', s == 200, f'({s})') + s, p, _, t = api('POST', f"/chats/{chat['id']}/attachments?filename=b.png", auth=auth, raw=SAMPLE, ctype='image/png') + check('upload -> 503 unavailable', s == 503 and err(p) == 'unavailable', f'({s} {err(p)}, {t:.2f} s)') + check('upload отказал быстрее timeout LB (< 8 s)', t < 8, f'({t:.2f} s)') + s, p, _, t = api('GET', f"/attachments/{att['id']}/content", auth=auth) + check('download -> 503 unavailable', s == 503 and err(p) == 'unavailable', f'({s} {err(p)}, {t:.2f} s)') + check('download отказал быстрее timeout LB (< 8 s)', t < 8, f'({t:.2f} s)') + s, p, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=c.png", auth=auth, raw=b'junk', ctype='image/png') + check('пользовательская ошибка при отказе S3 остаётся 415', s == 415, f'({s})') + intents = docker('exec', db, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', + 'SELECT count(*) FROM upload_intents') + print(f' upload_intents после неудачных upload: {intents}', flush=True) + finally: + docker('start', s3) + print('-- s3 запущен', flush=True) + + check('s3 снова healthy', wait_healthy(s3)) + deadline = time.monotonic() + 30 + while True: + s, new, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=d.png", auth=auth, raw=SAMPLE, ctype='image/png') + if s == 201 or time.monotonic() > deadline: + break + time.sleep(1) + check('после восстановления S3 upload -> 201', s == 201, f'({s})') + s, _, data, _ = api('GET', f"/attachments/{att['id']}/content", auth=auth) + check('старый файл скачивается побайтно', s == 200 and data == SAMPLE, f'({s})') + + print('PASS: bucket приватен вне API, отказ S3 даёт 503 только для вложений' if not failed + else f'FAIL: {failed} проверок не прошло') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/upload_compensation.py b/student-tests/upload_compensation.py new file mode 100644 index 00000000..46bba9de --- /dev/null +++ b/student-tests/upload_compensation.py @@ -0,0 +1,221 @@ +"""Уровень 4: отказ между S3 PUT и COMMIT не оставляет вечных объектов. + + 1. db_error_after_s3_put — процесс API (FAULT_INJECTION, порт 8091 внутри + контейнера реплики) теряет БД после PUT -> 503, фоновая компенсация + сама удаляет объект и intent; + 2. crash_after_s3_put — процесс (порт 8090) умирает после PUT до COMMIT -> + остаются intent и объект; janitor после grace удаляет оба; + 3. свежий intent (моложе grace, «загрузка в процессе») janitor не трогает; + готовое вложение после janitor скачивается побайтно. + + docker compose --profile tools run --rm tester python3 student-tests/upload_compensation.py +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/') +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +GRACE = 20 + +failed = 0 + + +def check(label, ok, detail=''): + global failed + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} {label} {detail}".rstrip(), flush=True) + + +def docker(*args, stdin=None) -> str: + r = subprocess.run(['docker', *args], capture_output=True, text=True, input=stdin) + if r.returncode: + raise RuntimeError(f'docker {args[0]} failed: {r.stderr.strip()[-300:]}') + return r.stdout.strip() + + +def container(service: str) -> str: + return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', f'label=com.docker.compose.service={service}').split()[0] + + +API = DB = '' + + +def psql(sql: str) -> str: + return docker('exec', DB, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', sql) + + +def intents() -> set[str]: + return set(filter(None, psql('SELECT object_key FROM upload_intents').splitlines())) + + +def s3_exists(*keys: str) -> list: + code = """ +import asyncio, json, sys +from app.storage import storage +async def main(): + await storage.start() + out = [await storage.exists(k) for k in json.load(sys.stdin)] + await storage.close() + print(json.dumps(out)) +asyncio.run(main()) +""" + return json.loads(docker('exec', '-i', API, 'python', '-c', code, stdin=json.dumps(list(keys)))) + + +def run_janitor(grace: int) -> int: + code = f""" +import asyncio +from app.attachments import cleanup_stale_uploads +from app.storage import storage +async def main(): + await storage.start() + print(await cleanup_stale_uploads({grace})) + await storage.close() +asyncio.run(main()) +""" + return int(docker('exec', API, 'python', '-c', code).splitlines()[-1]) + + +def http(method, url, data=None, headers=None, timeout=30): + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +class Client: + def __init__(self, base): + self.base = base + + def call(self, method, path, body=None, auth=None, raw=None, ctype=None): + h = {} + if auth: + h['Authorization'] = auth + if body is not None: + raw, ctype = json.dumps(body).encode(), 'application/json' + if ctype: + h['Content-Type'] = ctype + s, data = http(method, self.base + '/api/v1' + path, raw, h) + try: + return s, json.loads(data) + except ValueError: + return s, data + + def upload(self, auth, chat_id): + return self.call('POST', f'/chats/{chat_id}/attachments?filename=c.png', + auth=auth, raw=SAMPLE, ctype='image/png') + + +def start_faulty(mode: str, port: int) -> Client: + docker('exec', '-d', '-e', f'FAULT_INJECTION={mode}', '-e', f'APP_PORT={port}', + '-e', 'JANITOR_INTERVAL_SECONDS=3600', API, 'python', '-m', 'app.server') + name = docker('inspect', '-f', '{{.Name}}', API).lstrip('/') + base = f'http://{name}:{port}' + for _ in range(60): + try: + if http('GET', base + '/health/ready', timeout=2)[0] == 200: + return Client(base) + except OSError: + pass + time.sleep(0.5) + raise RuntimeError(f'faulty process {mode} did not start') + + +def stop_faulty() -> None: + """Kill the extra app.server processes (not PID 1, the real replica).""" + code = """ +import os, signal +for p in filter(str.isdigit, os.listdir('/proc')): + if int(p) in (1, os.getpid()): + continue + try: + cmd = open(f'/proc/{p}/cmdline', 'rb').read() + except OSError: + continue + if b'app.server' in cmd: + os.kill(int(p), signal.SIGKILL) +""" + docker('exec', API, 'python', '-c', code) + + +def main() -> int: + global API, DB + API, DB = container('api'), container('db') + lb = Client(BASE) + + name = 'comp' + uuid.uuid4().hex[:12] + password = 'compensation-password-1' + lb.call('POST', '/users', {'username': name, 'display_name': 'Comp', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + auth = 'Bearer ' + lb.call('POST', '/auth/sessions', auth=basic)[1]['token'] + me = lb.call('GET', '/users/me', auth=auth)[1] + chat = lb.call('POST', '/chats', {'title': 'compensation', 'member_ids': [me['id']]}, auth=auth)[1] + ready = lb.upload(auth, chat['id'])[1] + + try: + # 1. DB lost after PUT -> 503 + background compensation + before = intents() + s, _ = start_faulty('db_error_after_s3_put', 8091).upload(auth, chat['id']) + time.sleep(2) + check('db_error_after_s3_put: 503', s == 503, f'({s})') + check('db_error_after_s3_put: компенсация убрала intent', intents() - before == set()) + + # 2. crash after PUT -> intent + object stay + before = intents() + try: + s, _ = start_faulty('crash_after_s3_put', 8090).upload(auth, chat['id']) + result = f'HTTP {s}' + except OSError as exc: + result = type(exc).__name__ + crash_keys = sorted(intents() - before) + check('crash_after_s3_put: клиент не получил 201', '201' not in result, f'({result})') + check('crash_after_s3_put: остались intent и объект', + len(crash_keys) == 1 and s3_exists(*crash_keys) == [True]) + finally: + stop_faulty() + + check('janitor до grace ничего не удаляет', + run_janitor(GRACE) == 0 and set(crash_keys) <= intents()) + + print(f'-- ждём grace {GRACE + 1} s', flush=True) + time.sleep(GRACE + 1) + + # 3. fresh in-flight upload (younger than grace) must survive + inflight = f'originals/{uuid.uuid4().hex}' + psql(f"INSERT INTO upload_intents (object_key, attachment_id, chat_id) " + f"VALUES ('{inflight}', '{uuid.uuid4()}', '{chat['id']}')") + + cleaned = run_janitor(GRACE) + left = intents() + check('janitor после grace удалил intent от crash', cleaned >= 1 and not set(crash_keys) & left, + f'(cleaned={cleaned})') + check('janitor удалил объект от crash', s3_exists(*crash_keys) == [False]) + check('свежий intent не тронут', inflight in left) + psql(f"DELETE FROM upload_intents WHERE object_key = '{inflight}'") + + s, body = lb.call('GET', f"/attachments/{ready['id']}/content", auth=auth) + check('готовое вложение цело', s == 200 and hashlib.sha256(body).hexdigest() == ready['sha256']) + + print('PASS: компенсация и janitor не оставляют вечных объектов' if not failed + else f'FAIL: {failed} проверок не прошли') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/upload_failover.py b/student-tests/upload_failover.py new file mode 100644 index 00000000..63e6a959 --- /dev/null +++ b/student-tests/upload_failover.py @@ -0,0 +1,105 @@ +"""Upload -> остановка обслужившей реплики -> download через другую, для КАЖДОЙ реплики. + +Публичный failover-тест останавливает ту реплику, что ему ответила (при +round-robin — обычно одну и ту же). Здесь цель выбирается явно: файл +загружается, пока X-Instance-Id не совпадёт с целевой репликой, затем она +останавливается (docker stop, остаётся остановленной на время проверки), +файл скачивается через LB и сравнивается по SHA-256; реплика запускается снова. + + docker compose --profile tools run --rm tester python3 student-tests/upload_failover.py +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/') +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def docker(*args) -> str: + return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip() + + +def call(method, path, body=None, auth=None, raw=None, ctype=None): + h = {} + if auth: + h['Authorization'] = auth + if body is not None: + raw, ctype = json.dumps(body).encode(), 'application/json' + if ctype: + h['Content-Type'] = ctype + req = urllib.request.Request(BASE + '/api/v1' + path, data=raw, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read(), r.headers.get('X-Instance-Id') + except urllib.error.HTTPError as e: + return e.code, e.read(), e.headers.get('X-Instance-Id') + + +def main() -> int: + failed = 0 + replicas = docker('ps', '-q', '--no-trunc', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', 'label=com.docker.compose.service=api').split() + assert len(replicas) >= 2, 'need >= 2 running api replicas' + + name = 'ufo' + uuid.uuid4().hex[:12] + password = 'upload-failover-password' + call('POST', '/users', {'username': name, 'display_name': 'UFO', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + auth = 'Bearer ' + json.loads(call('POST', '/auth/sessions', auth=basic)[1])['token'] + me = json.loads(call('GET', '/users/me', auth=auth)[1]) + chat = json.loads(call('POST', '/chats', {'title': 'ufo', 'member_ids': [me['id']]}, auth=auth)[1]) + digest = hashlib.sha256(SAMPLE).hexdigest() + + for target in replicas: + alias = target[:12] + for _ in range(20): + s, body, instance = call('POST', f"/chats/{chat['id']}/attachments?filename=ufo.png", + auth=auth, raw=SAMPLE, ctype='image/png') + if s == 201 and target.startswith(instance): + break + else: + print(f'FAIL could not hit replica {alias}') + failed += 1 + continue + att = json.loads(body) + docker('stop', '--time', '15', target) + time.sleep(1) + try: + served = set() + ok = True + for _ in range(4): + s, data, instance = call('GET', f"/attachments/{att['id']}/content", auth=auth) + served.add(instance) + ok &= s == 200 and hashlib.sha256(data).hexdigest() == digest + ok &= alias not in served + failed += not ok + print(f"{'OK ' if ok else 'FAIL'} upload через {alias} -> {alias} остановлена -> " + f"download через {sorted(served)}: SHA-256 {'совпал' if ok else 'НЕ совпал'}", flush=True) + finally: + docker('start', target) + for _ in range(60): + if docker('inspect', '-f', '{{.State.Health.Status}}', target) == 'healthy': + break + time.sleep(1) + time.sleep(3) + + print('PASS: файл доступен после остановки любой обслужившей реплики' if not failed + else f'FAIL: {failed}') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..bb99a14d --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,230 @@ +"""Эксперимент: пик памяти API при N одновременных загрузках (stdlib + docker CLI). + +Гипотеза: память реплики ограничена конструкцией, а не числом/размером +загрузок: тело пишется в SpooledTemporaryFile (в RAM <= UPLOAD_SPOOL_MEMORY_BYTES +на загрузку), PNG проверяется потоково, одновременно <= MAX_CONCURRENT_UPLOADS +на реплику. Ожидаем прирост пика RSS << N x размер файла. + +Методика: перезапуск API-реплик (сброс high-water mark) -> базовые VmRSS/VmHWM -> +N потоков одновременно (Barrier) загружают валидный PNG из шума ~9.5 MiB -> +во время загрузок внутри каждой реплики каждые 100 мс читается VmRSS процесса +API (/proc/1/status) и anon cgroup (без page cache временных файлов) -> пик +минус базовое значение -> проверка SHA-256. VmHWM не годится: стартовый пик +импорта библиотек (~170 MiB) выше пика загрузок. + + docker compose --profile tools run --rm tester python3 student-tests/upload_memory.py --concurrency 12 +""" +from __future__ import annotations + +import argparse +import base64 +import collections +import hashlib +import json +import os +import struct +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +import zlib + +PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4') + + +def docker(*args) -> str: + return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip() + + +def service_containers(service: str) -> list[str]: + return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', f'label=com.docker.compose.service={service}').split() + + +def api_containers() -> list[str]: + return service_containers('api') + + +def memory(cid: str) -> dict: + out = docker('exec', cid, 'sh', '-c', + 'grep -E "^Vm(RSS|HWM)" /proc/1/status; ' + 'echo "peak $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0)"; ' + 'echo "current $(cat /sys/fs/cgroup/memory.current)"; ' + 'grep -E "^anon " /sys/fs/cgroup/memory.stat') + values = {} + for line in out.splitlines(): + name, value = line.replace(':', '').split()[:2] + values[name] = int(value) * (1024 if name.startswith('Vm') else 1) + return {k: round(v / 2 ** 20, 1) for k, v in values.items()} + + +SAMPLER = ('while :; do r=$(grep VmRSS /proc/1/status | tr -s " " | cut -d" " -f2); ' + 'a=$(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2); ' + 'echo "$r $a"; sleep 0.1; done') + + +class Sampler: + def __init__(self, cid: str): + self.proc = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', SAMPLER], + stdout=subprocess.PIPE, text=True) + self.rss = self.anon = 0 + self.samples = 0 + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + + def _read(self): + for line in self.proc.stdout: + try: + rss_kib, anon = map(int, line.split()) + except ValueError: + continue + self.rss = max(self.rss, rss_kib * 1024) + self.anon = max(self.anon, anon) + self.samples += 1 + + def stop(self) -> dict: + self.proc.terminate() + self.proc.wait() + return {'VmRSS_peak': round(self.rss / 2 ** 20, 1), 'anon_peak': round(self.anon / 2 ** 20, 1), + 'samples': self.samples} + + +def noise_png(target_bytes: int) -> bytes: + width = 1800 + height = target_bytes // (width * 3 + 1) + raw = b''.join(b'\x00' + os.urandom(width * 3) for _ in range(height)) + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'') + + +def call(base, method, path, body=None, auth=None, raw=None, ctype=None, timeout=120): + h = {} + if auth: + h['Authorization'] = auth + if body is not None: + raw, ctype = json.dumps(body).encode(), 'application/json' + if ctype: + h['Content-Type'] = ctype + req = urllib.request.Request(base + '/api/v1' + path, data=raw, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read(), r.headers.get('X-Instance-Id') + except urllib.error.HTTPError as e: + return e.code, e.read(), e.headers.get('X-Instance-Id') + + +def wait_ready(base, seconds=90): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base + '/health/ready', timeout=2) as r: + if r.status == 200: + return + except Exception: + pass + time.sleep(1) + raise SystemExit('API is not ready') + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://lb:8080')) + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--size-mib', type=float, default=9.5) + p.add_argument('--no-restart', action='store_true') + p.add_argument('--output') + args = p.parse_args() + base = args.base_url.rstrip('/') + + data = noise_png(int(args.size_mib * 2 ** 20)) + digest = hashlib.sha256(data).hexdigest() + print(f'file: {len(data)} bytes ({len(data) / 2 ** 20:.2f} MiB), concurrency {args.concurrency}', flush=True) + + containers = api_containers() + if not args.no_restart: + docker('restart', '--time', '15', *containers) + time.sleep(3) + wait_ready(base) + time.sleep(3) + + name = 'mem' + uuid.uuid4().hex[:12] + password = 'upload-memory-password' + call(base, 'POST', '/users', {'username': name, 'display_name': 'Mem', 'password': password}) + basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode() + s, session, _ = call(base, 'POST', '/auth/sessions', auth=basic) + auth = 'Bearer ' + json.loads(session)['token'] + user = json.loads(call(base, 'GET', '/users/me', auth=auth)[1]) + chat = json.loads(call(base, 'POST', '/chats', {'title': 'mem', 'member_ids': [user['id']]}, auth=auth)[1]) + + before = {c[:12]: memory(c) for c in containers} + samplers = {c[:12]: Sampler(c) for c in containers} + s3 = service_containers('s3')[0] + s3_before = memory(s3) + s3_sampler = Sampler(s3) + time.sleep(1) + barrier = threading.Barrier(args.concurrency) + results = [] + lock = threading.Lock() + + def worker(): + barrier.wait() + t = time.monotonic() + s, body, instance = call(base, 'POST', f"/chats/{chat['id']}/attachments?filename=noise.png", + auth=auth, raw=data, ctype='image/png') + with lock: + results.append((s, time.monotonic() - t, instance, body)) + + started = time.monotonic() + threads = [threading.Thread(target=worker) for _ in range(args.concurrency)] + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - started + time.sleep(0.5) + peaks = {c: sm.stop() for c, sm in samplers.items()} + s3_peak = s3_sampler.stop() + after = {c[:12]: memory(c) for c in containers} + + ok = [r for r in results if r[0] == 201] + sha_ok = all(json.loads(r[3])['sha256'] == digest for r in ok) + if ok: + att = json.loads(ok[0][3]) + s, content, _ = call(base, 'GET', f"/attachments/{att['id']}/content", auth=auth) + sha_ok = sha_ok and hashlib.sha256(content).hexdigest() == digest + latencies = sorted(r[1] for r in ok) + summary = { + 'concurrency': args.concurrency, + 'file_bytes': len(data), + 'statuses': dict(collections.Counter(r[0] for r in results)), + 'per_instance': dict(collections.Counter(r[2] for r in results)), + 'wall_seconds': round(wall, 2), + 'latency_p50_s': round(latencies[len(latencies) // 2], 2) if latencies else None, + 'latency_max_s': round(latencies[-1], 2) if latencies else None, + 'sha256_ok': sha_ok, + 'memory_mib_before': before, + 'memory_mib_peak_during': peaks, + 'memory_mib_after': after, + 'rss_peak_increase_mib': { + c: round(peaks[c]['VmRSS_peak'] - before[c]['VmRSS'], 1) for c in peaks + }, + 'anon_peak_increase_mib': { + c: round(peaks[c]['anon_peak'] - before[c]['anon'], 1) for c in peaks + }, + 's3_server_mib': {'VmRSS_before': s3_before['VmRSS'], **s3_peak}, + 'naive_in_memory_estimate_mib': round(args.concurrency * len(data) / 2 ** 20, 1), + } + text = json.dumps(summary, ensure_ascii=False, indent=2) + print(text) + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(text + '\n') + return 0 if ok and sha_ok and all(r[0] in (201, 429) for r in results) else 1 + + +if __name__ == '__main__': + raise SystemExit(main())