Done Track 4
This commit is contained in:
@@ -1,20 +0,0 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
@@ -32,3 +32,4 @@ coverage/
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
# Local secrets (scripts/init-secrets.sh)
|
||||
infra/secrets/*.txt
|
||||
ЗАЩИТА.md
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
# Messenger API (lab3). One image for the "api" and "migrate" services.
|
||||
# Messenger API (lab4). One image for the "api", "migrate" and "s3-init" services.
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
@@ -10,7 +10,7 @@ COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
COPY scripts/migrate.py ./scripts/migrate.py
|
||||
COPY scripts/migrate.py scripts/s3_bootstrap.py ./scripts/
|
||||
COPY migrations ./migrations
|
||||
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Teaching outline, not a runnable Dockerfile. Implement your own Dockerfile.
|
||||
# 1. Choose a runtime/toolchain version and pin the base image tag/digest.
|
||||
# 2. Copy dependency manifests first; install from the language lockfile.
|
||||
# 3. Copy source and build. For compiled languages use a separate runtime stage.
|
||||
# 4. Run the HTTP process as PID 1; listen on 0.0.0.0:8080.
|
||||
# 5. Add non-root, signals and healthcheck according to the current rubric.
|
||||
# 6. Pass runtime config/secrets at startup, never COPY .env or secret files.
|
||||
# This file deliberately has no FROM/CMD: selecting/implementing them is the task.
|
||||
@@ -1,37 +1,137 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Максимова, БВТ2303 (TODO: полное ФИО)
|
||||
- Целевая оценка; выбранный трек 5: 4 (трек не выбирается)
|
||||
- Commit/PR: TODO после публикации (ветка `solution/maksimova_bvt2303`)
|
||||
- Использованные источники, библиотеки и помощь ИИ: FastAPI, SQLAlchemy/asyncpg, argon2-cffi, redis-py, aiobotocore/botocore, Pillow (только для JPEG); SeaweedFS, PostgreSQL, Valkey, HAProxy. Документация: AWS S3 PutObject/SigV4, SeaweedFS wiki (S3 API, `s3.json` identities), спецификация PNG (RFC 2083), HAProxy 3.0 configuration manual. Помощь ИИ (Claude Code): ревью и доработка кода вложений, написание тестов в `student-tests/`, проведение экспериментов и черновик этого отчёта; все результаты получены реальными прогонами на стенде и проверены.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Проверено на: Windows 11 + Docker Desktop 29.3.0 (WSL2, kernel 6.6.87, 14 vCPU, 7.4 GiB RAM), Docker Compose 5.1.0, Python 3.13 на хосте (для `make test`; достаточно 3.9+).
|
||||
|
||||
| Роль т | Service т | Образ | Health / readiness | Volume | Сеть / порт | Credentials |
|
||||
|---------------------|-----------|--------------------------------------------|-----------------------------------------------------------|-----------------------------------------|------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------|
|
||||
| LB | `lb` | `haproxy:3.0.11-alpine` | `/lb-health`; backend-check `GET /health/ready` раз в 1 s | — | edge; `${LB_PORT:-8080}` на хосте, stats только `127.0.0.1:8404` | — |
|
||||
| API × 2 | `api` | сборка `Dockerfile` (`python:3.12.7-slim`) | `/health/ready` (PostgreSQL + не в drain) | нет | edge + backend, порт не публикуется | Docker secrets: `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` |
|
||||
| PostgreSQL | `db` | `postgres:16.4-alpine` | `pg_isready` | `postgres_data` | backend (internal) | `postgres_password` |
|
||||
| Valkey (кэш сессий) | `valkey` | `valkey/valkey:8.0.1-alpine` | `PING` | нет (кэш, источник истины — PostgreSQL) | backend | `valkey_password` |
|
||||
| S3 | `s3` | `chrislusf/seaweedfs:3.97` | `GET :9000/healthz` | `s3_data` | backend, порт не публикуется; master/filer/volume слушают только `127.0.0.1` внутри контейнера | admin + API ключи из secrets подставляются в `s3.json` при старте |
|
||||
| Bootstrap bucket | `s3-init` | образ API | one-shot, `service_completed_successfully` | — | backend | единственный, кто получает admin-ключ S3 |
|
||||
| Миграции | `migrate` | образ API | one-shot | — | backend | `database_url` |
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh # создаёт infra/secrets/*.txt (случайные значения, не перезаписывает, не печатает)
|
||||
cp .env.example .env # при занятом 8080 задайте LB_PORT=8081
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# порядок: db, valkey, s3 -> healthy -> migrate (SQL migrations/*.sql, повтор = SKIP)
|
||||
# s3-init (bucket + самопроверки приватности/прав, повтор = keep) -> api x2 healthy -> lb
|
||||
curl -fsS http://localhost:${LB_PORT:-8080}/health/ready
|
||||
make test BASE_URL=http://localhost:${LB_PORT:-8080} # на Windows: PYTHONUTF8=1 python tests/smoke.py --base-url ...
|
||||
|
||||
# hook-тесты и собственные проверки — в Linux-контейнере tester (docker CLI + docker.sock):
|
||||
docker compose --profile tools run --rm tester make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://lb:8080
|
||||
docker compose --profile tools run --rm tester make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://lb:8080
|
||||
docker compose --profile tools run --rm tester sh scripts/start-api.sh # вернуть остановленную реплику
|
||||
docker compose --profile tools run --rm tester python3 student-tests/<test>.py
|
||||
|
||||
docker compose stop # безопасная остановка, данные сохраняются
|
||||
docker compose down -v # НАМЕРЕННОЕ удаление всех учебных данных (volumes)
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
client -> HAProxy (lb) -> api x2 --+--> PostgreSQL: users, chats, messages, sessions, idempotency_keys,
|
||||
| attachments (метаданные), message_attachments, upload_intents
|
||||
+--> Valkey: кэш сессий (tombstone при logout)
|
||||
+--> SeaweedFS S3: приватный bucket messenger-attachments/originals/<random hex>
|
||||
```
|
||||
|
||||
Перенесено из lab3 без изменения контракта: регистрация/Basic→Bearer, сессии в PostgreSQL с кэшем в Valkey, идемпотентная отправка сообщений, `X-Instance-Id`, graceful drain, HAProxy с active check, hooks `restart.sh`/`stop-one.sh`. Изменено: `CreateMessage` принимает `attachment_ids` (по умолчанию `[]`) и `text=""` при наличии вложений; fingerprint идемпотентности без вложений совпадает с lab3, так что старые ключи повторяются корректно.
|
||||
|
||||
Модель. `attachments` (id, chat_id, owner_id, filename — только для отображения, content_type, size, sha256, width/height, status=`ready`, `object_key UNIQUE`) привязано к чату при загрузке. `message_attachments(message_id, attachment_id, position)` хранит порядок; одно вложение можно сослать из нескольких сообщений чата. Миграция `004_attachments.sql` — expand-only.
|
||||
|
||||
Upload (`app/attachments.py`), всё до сохранения байтов:
|
||||
1. filename (400: `/`, `\`, NUL, Cc/Cf/bidi, пустое/только точки, >255) → членство в чате (404) → `Content-Type` ровно `image/png|image/jpeg` (415, multipart не принимается) → `Content-Length > 10 MiB` → 413 **без чтения тела**;
|
||||
2. слот загрузки: ≤ `MAX_CONCURRENT_UPLOADS=8` на реплику, ожидание ≤10 s, иначе 429;
|
||||
3. тело читается потоково в `SpooledTemporaryFile` (в RAM ≤ 1 MiB на загрузку), считается размер и SHA-256 на лету; превышение 10 MiB в chunked → 413; приём дольше 60 s → 400;
|
||||
4. проверка содержимого в потоке: формат по magic bytes и совпадение с заявленным MIME; PNG — собственный потоковый валидатор (CRC каждого чанка, IHDR, распаковка IDAT кусками по 64 KiB с подсчётом и сравнением с ожидаемым размером, без хранения пикселей); лимит 40 млн пикселей проверяется по IHDR/SOF **до** распаковки (413); JPEG — Pillow `draft()` 1/8 + `load()` (повреждённый поток → 415);
|
||||
5. `INSERT upload_intent` + COMMIT → S3 PUT (SigV4 подписывает SHA-256 тела) → одна транзакция «DELETE intent + INSERT attachment» → **201 ready**. Честный 201 возможен только после этого COMMIT: и байты, и метаданные уже устойчивы.
|
||||
|
||||
Download: авторизация по `chat_members` в PostgreSQL (чужое/неизвестное → 404), затем потоковая отдача из S3 через API, без redirect, с истинным `Content-Type`, `Content-Length`, `nosniff`, `Cache-Control: private, no-store`. Приватный bucket не заменяет ACL: случайный ключ можно утечь через лог, а права участника чата знает только приложение.
|
||||
|
||||
Согласованность БД/S3 (ответ на «PUT прошёл, а COMMIT нет»):
|
||||
|
||||
| Отказ | Что делает система |
|
||||
|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| PUT упал / истёк дедлайн | 503 сразу. Исход PUT неизвестен (S3 может дописать объект **после** нашего DELETE), поэтому объект удаляется best effort в фоне, а intent **остаётся**; janitor удалит объект ещё раз после grace и только потом intent |
|
||||
| COMMIT упал | 503 сразу. Исход COMMIT тоже неизвестен (мог пройти, а потерялся ответ). Фоновая компенсация берёт `SELECT … FOR UPDATE` intent: intent нет → COMMIT прошёл, объект не трогаем; иначе DELETE объекта → DELETE intent |
|
||||
| Процесс умер после PUT | intent остался; janitor каждой реплики (`JANITOR_INTERVAL_SECONDS=60`, jitter) через `UPLOAD_GRACE_SECONDS=900` удаляет объект и intent; `FOR UPDATE SKIP LOCKED` — реплики не мешают друг другу и финализирующейся загрузке |
|
||||
| Строка без объекта (потеря байтов вне приложения) | download → 500 `internal_error` и ERROR в логе; метаданные не удаляются (сообщения на них ссылаются) |
|
||||
|
||||
Инвариант: объект без строки существует, только пока жив его intent; объект закоммиченной строки никогда не удаляется. Grace 900 s ≫ максимального времени upload (60 s приём + 16 s PUT).
|
||||
|
||||
Недоступность S3. `/health/ready` S3 не проверяет: без S3 деградируют только вложения (503), чаты и сообщения работают; иначе отказ S3 вывел бы из LB все реплики. Каждый S3-вызов ограничен общим дедлайном `S3_OPERATION_TIMEOUT_SECONDS=6` (DNS + повторы) — меньше `timeout server 10s` у LB; для PUT дедлайн `6 s + size / 1 MiB/s` (≤16 s для 10 MiB), а HAProxy даёт POST вложения 40 s. L7-повторы HAProxy для `/api/v1/attachments/` отключены: 503 там означает отказ общего S3, повтор на другой реплике бесполезен.
|
||||
|
||||
Права S3 (`infra/seaweedfs/s3.json.template`): identity `admin` есть только у `s3-init`; identity `messenger-api` — `Read/Write/List` только на `messenger-attachments` (Write нужен для DELETE janitor-а). Анонимного доступа нет. Admin UI/API SeaweedFS (master 9333, filer 8888, volume 8080, gRPC) слушают только `127.0.0.1` внутри контейнера.
|
||||
|
||||
*Соответствие `.env.example`:** `APP_HOST/APP_PORT`, `SESSION_TTL_SECONDS`, `DATABASE_URL_FILE`, `SESSION_STORE=valkey`, `CACHE_URL_FILE`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS`, `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`, `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` читаются `app/config.py` под теми же именами. Дополнительно в `compose.yaml` (`x-api-env`): `S3_OBJECT_PREFIX`, `S3_CONNECT/READ/OPERATION_TIMEOUT_SECONDS`, `S3_PUT_MIN_BYTES_PER_SECOND`, `UPLOAD_SPOOL_MEMORY_BYTES`, `UPLOAD_TIMEOUT_SECONDS`, `MAX_CONCURRENT_UPLOADS`, `UPLOAD_SLOT_WAIT_SECONDS`, `UPLOAD_GRACE_SECONDS`, `JANITOR_INTERVAL_SECONDS`, `FAULT_INJECTION` (только для экспериментов, пусто в норме). `APP_ENV` не используется.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Все файлы — в `evidence/`, тесты — в `student-tests/`; прогоны 2026-10-30 на стенде выше.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
|-----------------------------------------------------------------------------------------------|-------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------|
|
||||
| Публичный smoke | `make test BASE_URL=http://localhost:8081` | `PASS: lab4 public API scenario` (SHA-256, 404 постороннему) | `smoke.txt` |
|
||||
| 3: S3 в стенде, метаданные в PG, байты в приватном bucket | `student-tests/s3_failures.py`; `api-no-local-files.txt` | object_key только в PG; анонимные GET/HEAD/LIST/PUT/DELETE напрямую в S3 → 403; у API нет volumes, `/tmp` пуст | `s3-failures.txt`, `api-no-local-files.txt` |
|
||||
| 3: PNG/JPEG ≤10 MiB, ≤40 Мпикс, MIME по содержимому, опасные filename | `student-tests/attachment_edge_cases.py` | 60/60 OK: ровно 10 MiB и ровно 40 Мпикс → 201; +1 байт → 413; бомба 50 Мпикс (~10 KB) → 413 | `attachment-edge-cases.txt` |
|
||||
| 3: upload → стоп обслужившей реплики → download через другую | `make test-failover` ×2; `student-tests/upload_failover.py` | failover: 0 ошибок, восстановление записи 4.0/4.2 s; для **каждой** из двух реплик SHA-256 совпал после её остановки | `failover.txt`, `upload-failover.txt` |
|
||||
| 3: restart с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; те же ID контейнеров db/s3/valkey/api/lb с новым `StartedAt`, volumes на месте | `persistence.txt` |
|
||||
| 4: тесты большого файла, повреждения, подмены MIME, чужого id, вложения другого чата; нет 500 | `attachment_edge_cases.py` | обрезка/CRC/мусор в zlib при верном CRC/без IEND → 415; PNG как JPEG, JPEG как PNG, text/plain, multipart → 415; вложение чужого и своего другого чата → 404; ни одного 500 | `attachment-edge-cases.txt` |
|
||||
| 4: компенсация/сверка/очистка с grace | `student-tests/upload_compensation.py` | DB error после PUT → 503 + фоновая компенсация, intent не остался; crash после PUT → intent + объект, janitor до grace их не трогает, после grace удаляет оба; свежий intent и готовое вложение целы | `upload-compensation.txt` |
|
||||
| 4: лимиты RAM, ≥10 одновременных загрузок | `student-tests/upload_memory.py --concurrency 12/20` | 12 × 9.5 MiB: прирост пика RSS 13–14 MiB на реплику (наивно 57 MiB данных на реплику) | `upload-memory-c12.json`, `upload-memory-c20.json` |
|
||||
| 4: идемпотентный bootstrap, отдельные ограниченные credentials | `docker compose run --rm s3-init` ×2, `migrate` | `keep bucket`, все самопроверки PASS: API-ключ не создаёт/не удаляет bucket, не видит чужих; миграции SKIP | `bootstrap-idempotent.txt` |
|
||||
| 4: админ-консоль закрыта, private bucket проверен вне API | `s3_failures.py` из сети стенда | порты 9333/8888/8080/19333/18888/23646 недоступны; S3 не опубликован на хосте; анонимные запросы → 403 | `s3-failures.txt` |
|
||||
| 3/4: недоступность S3 | `s3_failures.py` (docker stop s3) | ready 200, сообщения 201, metadata 200; upload/download → 503 за 6.0–6.2 s; 415 остаётся 415; после старта S3 всё работает, старый файл цел | `s3-failures.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Память при одновременных загрузках (уровень 4)
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Гипотеза: пик памяти API определяется конструкцией (spool ≤1 MiB/загрузку, потоковая проверка PNG, ≤8 слотов), а не числом × размером файлов.
|
||||
Методика: `upload_memory.py` перезапускает обе реплики (сброс состояния), N потоков стартуют одновременно (Barrier) и загружают валидный PNG из шума 9 962 537 байт (9.5 MiB, несжимаемый — худший случай для проверки IDAT); внутри каждой реплики и S3 каждые 100 мс читаются `VmRSS` процесса и `anon` cgroup. `VmHWM` не подходит: пик импорта библиотек при старте (~170 MiB) выше пика загрузок. Лимиты: API 0.5 CPU / 256 MiB, S3 1 CPU / 1 GiB.
|
||||
|
||||
| N (на реплику) | Статусы | Wall, s | p50 / max, s | Пик RSS API − базовый (≈105 MiB) | Наивная оценка «всё в RAM» на реплику | Пик RSS SeaweedFS |
|
||||
|------------------------|-----------|-----------|----------------|------------------------------------|-----------------------------------------|---------------------|
|
||||
| 12 (6+6) | 12 × 201 | 5.2 | 5.0 / 5.2 | +12.9 / +13.7 MiB | 57 MiB | 546 MiB |
|
||||
| 20 (10+10, > 8 слотов) | 20 × 201 | 9.9 | 8.6 / 9.9 | +16.5 / +17.7 MiB | 95 MiB | 779 MiB |
|
||||
|
||||
Гипотеза для API подтверждена: прирост ~2 MiB на одновременную загрузку и почти не растёт сверх 8 слотов (лишние ждут слот; в одном из повторов при N=20 двое получили 429 после 10 s ожидания — ожидаемый backpressure). SHA-256 проверен.
|
||||
|
||||
Побочная находка: узким местом оказался S3-сервер. С исходным лимитом 512 MiB SeaweedFS при 12 параллельных PUT дошёл до 519 MiB и был убит (контейнер перезапущен, все загрузки — 503). Исправлено: лимит 1 GiB и `GOMEMLIMIT=600MiB` (мягкий лимит Go GC); после нагрузки RSS возвращается к ~210 MiB. Это одно измерение на одном ноутбуке, а не ёмкость SeaweedFS вообще.
|
||||
|
||||
### Компенсация отказа между S3 и БД (уровень 4)
|
||||
|
||||
Найденный дефект (до исправления). Сверка содержимого bucket со строками БД на «грязном» стенде нашла 12 объектов без строки и без intent — то, что запрещено инвариантом. Причина: при истёкшем дедлайне PUT компенсация удаляла объект **и intent**, а SeaweedFS дописывал PUT после DELETE. Второй риск того же рода: при неоднозначном COMMIT немедленное удаление объекта оставило бы строку без байтов. Исправлено (см. таблицу отказов выше); 12 лишних объектов удалены. `upload_compensation.py` после исправления проверяет компенсацию при ошибке БД и очистку janitor-ом после падения процесса.
|
||||
|
||||
### Отказы
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
|-----------------------------------------------------------------|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|
|
||||
| `docker stop` обслужившей API-реплики (×2 + для каждой реплики) | запись и download через другую | 0 ошибок записи; файл скачан по SHA-256 через оставшуюся | первая успешная запись через 4.0–4.2 s; потерь нет |
|
||||
| Restart lb + api + db + valkey + s3 с volumes | данные, сессия и оригинал сохранены | PASS с прежним токеном | hook и ожидание готовности уложились в лимиты теста (90 + 90 s), длительность отдельно не замерялась; потерь нет |
|
||||
| `docker stop s3` | только вложения → 503, остальное работает | ready 200, message 201, metadata 200, upload/download 503 за ~6 s | сразу после старта S3; потерь нет, intents неудачных PUT убирает janitor |
|
||||
| Падение процесса после PUT до COMMIT | клиент без 201, orphan не вечный | intent + объект → удалены janitor после grace | grace (900 s; 20 s в тесте) |
|
||||
| Потеря БД после PUT | 503, без мусора | 503; фоновая компенсация удалила объект и intent | ~0.1 s |
|
||||
| 12 параллельных PUT по 10 MiB при лимите S3 512 MiB | — | SeaweedFS убит по памяти, все 503 | исправлено лимитом 1 GiB + `GOMEMLIMIT` |
|
||||
| Оборванная клиентом загрузка | реплика продолжает работу | следующая загрузка 201; по коду PUT начинается только после приёма всего тела, поэтому объекта нет (отдельно не измерялось) | — |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker host, один PostgreSQL, один SeaweedFS без репликации, один LB: выдерживается отказ одной API-реплики и перезапуск хранилищ с volumes, но не потеря host/диска/S3-тома. Резервного копирования нет.
|
||||
- Нет отдельного reconciler: объект без строки и без intent (ручное вмешательство, баг) автоматически не удаляется; потеря байтов готового вложения видна только по 500 и ERROR в логе при download.
|
||||
- Прогрессивные JPEG > 16 Мпикс проверяются только по структуре маркеров (полное декодирование держало бы все DCT-коэффициенты в памяти); такие повреждения внутри энтропийного потока не ловятся.
|
||||
- `MAX_CONCURRENT_UPLOADS` — на реплику, не глобально; общий предел S3 задаётся числом реплик × 8. Ёмкость SeaweedFS при большем числе реплик не измерялась.
|
||||
- Измерения памяти — по одному-два прогона на ноутбуке; при N=12 в разных прогонах прирост был от 3.6 до 13.7 MiB на реплику, абсолютные числа зависят от аллокатора и версии Python.
|
||||
|
||||
@@ -0,0 +1,441 @@
|
||||
"""Attachments: synchronous upload to S3, metadata in PostgreSQL (lab4).
|
||||
|
||||
Upload, in this order (all checks before the bytes are stored):
|
||||
|
||||
1. filename (400) -> chat membership (404) -> Content-Type header (415)
|
||||
-> Content-Length (413 without reading the body);
|
||||
2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429;
|
||||
3. the body is read in chunks into a SpooledTemporaryFile (only
|
||||
UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more
|
||||
than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400;
|
||||
4. content validation in a thread (app.images): format by magic bytes,
|
||||
pixel limit from the header (413), integrity (415);
|
||||
5. upload intent COMMIT -> S3 PUT -> one transaction "delete intent +
|
||||
insert ready attachment" -> 201.
|
||||
|
||||
Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs
|
||||
in the background):
|
||||
- PUT failed or timed out: the outcome is unknown - S3 may still finish
|
||||
writing the object AFTER our DELETE. So the object is deleted best
|
||||
effort, but the intent is KEPT; the janitor deletes the object again
|
||||
after UPLOAD_GRACE_SECONDS and only then the intent;
|
||||
- COMMIT failed: the outcome is unknown too - COMMIT may have succeeded
|
||||
and only the answer was lost. Under the intent's row lock: no intent ->
|
||||
the row is committed, keep the object; else delete object, then intent;
|
||||
- the process died after PUT: the intent stays; the janitor deletes the
|
||||
object and the intent after UPLOAD_GRACE_SECONDS.
|
||||
So an object without a row exists only while its intent exists, and an
|
||||
object of a committed row is never deleted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from app import repositories
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.errors import (
|
||||
ApiError,
|
||||
InvalidRequest,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
RateLimited,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
from app.images import InvalidImage, TooManyPixels, validate_image
|
||||
from app.storage import ObjectMissing, StorageUnavailable, storage
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.attachments")
|
||||
|
||||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||||
# Characters that can turn a display name into a path or hide its real
|
||||
# extension: separators, NUL/control characters, bidi overrides etc.
|
||||
FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
|
||||
|
||||
upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
|
||||
# Strong references to background compensation tasks (asyncio keeps only
|
||||
# weak ones); if the process dies first, the intent stays for the janitor.
|
||||
_compensations: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def _to_attachment(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> str:
|
||||
if not 1 <= len(filename) <= 255 or not filename.strip():
|
||||
raise InvalidRequest("filename must be 1-255 characters, not blank.")
|
||||
|
||||
if "/" in filename or "\\" in filename:
|
||||
raise InvalidRequest("filename must not contain path separators.")
|
||||
|
||||
if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename):
|
||||
raise InvalidRequest("filename must not contain control characters.")
|
||||
|
||||
if filename.strip(". ") == "":
|
||||
raise InvalidRequest("filename must not be a relative path.")
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def declared_content_type(request: Request) -> str:
|
||||
raw = request.headers.get("Content-Type", "")
|
||||
media_type = raw.split(";", 1)[0].strip().lower()
|
||||
|
||||
if media_type not in ALLOWED_CONTENT_TYPES:
|
||||
raise UnsupportedMediaType(
|
||||
"Content-Type must be image/png or image/jpeg (raw bytes, not multipart)."
|
||||
)
|
||||
|
||||
return media_type
|
||||
|
||||
|
||||
def _too_large(message: str) -> PayloadTooLarge:
|
||||
error = PayloadTooLarge(message)
|
||||
# The rest of the body is not read: the connection cannot be reused.
|
||||
error.headers["Connection"] = "close"
|
||||
return error
|
||||
|
||||
|
||||
def check_declared_length(request: Request) -> None:
|
||||
raw = request.headers.get("Content-Length")
|
||||
if raw is None:
|
||||
return # chunked: the limit is enforced while reading
|
||||
|
||||
if not raw.isdigit():
|
||||
raise InvalidRequest("Invalid Content-Length.")
|
||||
|
||||
if int(raw) > settings.max_upload_bytes:
|
||||
raise _too_large("File is larger than 10 MiB.")
|
||||
|
||||
|
||||
async def _receive_body(request: Request, spool) -> tuple[int, str]:
|
||||
"""Stream the body into the spool; return (size, sha256)."""
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(settings.upload_timeout_seconds):
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise _too_large("File is larger than 10 MiB.")
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
raise ApiError(
|
||||
400,
|
||||
"invalid_request",
|
||||
"Upload was not completed in time.",
|
||||
{"Connection": "close"},
|
||||
) from None
|
||||
except ClientDisconnect:
|
||||
# Nobody will read this answer; nothing was stored anywhere.
|
||||
logger.info("upload interrupted by the client after %d bytes", size)
|
||||
raise InvalidRequest("Upload was interrupted.") from None
|
||||
|
||||
return size, digest.hexdigest()
|
||||
|
||||
|
||||
async def _acquire_slot() -> None:
|
||||
try:
|
||||
async with asyncio.timeout(settings.upload_slot_wait_seconds):
|
||||
await upload_slots.acquire()
|
||||
except TimeoutError:
|
||||
raise RateLimited("Too many concurrent uploads, retry later.", 2) from None
|
||||
|
||||
|
||||
async def _discard_after_failed_put(object_key: str) -> None:
|
||||
"""Best effort only; the intent stays for the janitor (late PUT)."""
|
||||
try:
|
||||
await storage.delete(object_key)
|
||||
except StorageUnavailable:
|
||||
logger.warning("compensation: S3 delete failed, janitor will retry")
|
||||
|
||||
|
||||
async def _compensate_failed_commit(object_key: str) -> None:
|
||||
"""Delete the object only if its upload really did not commit."""
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
row = await repositories.lock_upload_intent(db, object_key)
|
||||
if row is None:
|
||||
# COMMIT succeeded after all (or the janitor finished it).
|
||||
await db.rollback()
|
||||
return
|
||||
if not row["committed"]:
|
||||
try:
|
||||
await storage.delete(object_key)
|
||||
except StorageUnavailable:
|
||||
await db.rollback()
|
||||
logger.warning("compensation: S3 delete failed, janitor will retry")
|
||||
return
|
||||
await repositories.delete_upload_intent(db, object_key)
|
||||
except Exception as exc:
|
||||
logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__)
|
||||
|
||||
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
filename: str,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
validate_filename(filename)
|
||||
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db, chat_id=chat_id, user_id=current_user["id"]
|
||||
)
|
||||
if chat is None:
|
||||
raise NotFound()
|
||||
# Do not keep a pooled connection idle in a transaction while the
|
||||
# client uploads up to 10 MiB.
|
||||
await db.rollback()
|
||||
|
||||
content_type = declared_content_type(request)
|
||||
check_declared_length(request)
|
||||
|
||||
await _acquire_slot()
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(
|
||||
max_size=settings.upload_spool_memory_bytes
|
||||
) as spool:
|
||||
size, sha256 = await _receive_body(request, spool)
|
||||
if size == 0:
|
||||
raise InvalidRequest("Request body with image bytes is required.")
|
||||
|
||||
try:
|
||||
info = await asyncio.to_thread(
|
||||
validate_image, spool, content_type, settings.max_image_pixels
|
||||
)
|
||||
except TooManyPixels:
|
||||
raise PayloadTooLarge("Image has more than 40 million pixels.") from None
|
||||
except InvalidImage as exc:
|
||||
raise UnsupportedMediaType(str(exc)) from None
|
||||
|
||||
return await _store(
|
||||
db,
|
||||
spool=spool,
|
||||
chat_id=chat_id,
|
||||
owner_id=current_user["id"],
|
||||
filename=filename,
|
||||
content_type=info.content_type,
|
||||
size=size,
|
||||
sha256=sha256,
|
||||
width=info.width,
|
||||
height=info.height,
|
||||
)
|
||||
finally:
|
||||
upload_slots.release()
|
||||
|
||||
|
||||
async def _store(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
spool,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> dict[str, Any]:
|
||||
attachment_id = uuid4()
|
||||
# Server-generated random key, unrelated to the filename and to the
|
||||
# public attachment ID.
|
||||
object_key = f"{settings.s3_object_prefix}{uuid4().hex}"
|
||||
|
||||
await repositories.create_upload_intent(
|
||||
db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id
|
||||
)
|
||||
|
||||
try:
|
||||
await storage.put(object_key, spool, size, content_type)
|
||||
except StorageUnavailable as exc:
|
||||
logger.warning("S3 PUT failed: %s", exc)
|
||||
_in_background(_discard_after_failed_put(object_key))
|
||||
raise Unavailable() from None
|
||||
|
||||
if settings.fault_injection == "crash_after_s3_put":
|
||||
logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT")
|
||||
os._exit(1)
|
||||
|
||||
try:
|
||||
if settings.fault_injection == "db_error_after_s3_put":
|
||||
raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT")
|
||||
|
||||
row = await repositories.finalize_attachment(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=owner_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=size,
|
||||
sha256=sha256,
|
||||
width=width,
|
||||
height=height,
|
||||
object_key=object_key,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__)
|
||||
_in_background(_compensate_failed_commit(object_key))
|
||||
raise Unavailable() from None
|
||||
|
||||
if row is None:
|
||||
# The janitor already claimed this intent (upload slower than the
|
||||
# grace period): the object is (being) deleted, report a failure.
|
||||
logger.warning("upload intent expired before COMMIT")
|
||||
raise Unavailable()
|
||||
|
||||
return _to_attachment(row)
|
||||
|
||||
|
||||
async def get_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
row = await repositories.get_accessible_attachment(
|
||||
db, attachment_id=attachment_id, user_id=current_user["id"]
|
||||
)
|
||||
if row is None:
|
||||
raise NotFound()
|
||||
return _to_attachment(row)
|
||||
|
||||
|
||||
async def open_attachment_content(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
variant: str,
|
||||
current_user: dict[str, Any],
|
||||
) -> tuple[AsyncIterator[bytes], dict[str, str]]:
|
||||
"""Authorize in PostgreSQL, then stream the bytes from S3 (no redirect)."""
|
||||
row = await repositories.get_accessible_attachment(
|
||||
db, attachment_id=attachment_id, user_id=current_user["id"]
|
||||
)
|
||||
if row is None:
|
||||
raise NotFound()
|
||||
await db.rollback()
|
||||
|
||||
if variant != "original":
|
||||
# Lab4 has no variants; a ready attachment without it -> 404.
|
||||
raise NotFound()
|
||||
|
||||
try:
|
||||
chunks, length = await storage.open_download(row["object_key"])
|
||||
except StorageUnavailable as exc:
|
||||
logger.warning("S3 GET failed: %s", exc)
|
||||
raise Unavailable() from None
|
||||
except ObjectMissing:
|
||||
# Metadata says "ready" but the bytes are gone: an inconsistency,
|
||||
# not a user error (logged as ERROR).
|
||||
logger.error("object of attachment %s is missing in S3", attachment_id)
|
||||
raise
|
||||
|
||||
if length != row["size_bytes"]:
|
||||
logger.error("object of attachment %s has a wrong size", attachment_id)
|
||||
raise ObjectMissing(row["object_key"])
|
||||
|
||||
headers = {
|
||||
"Content-Type": row["content_type"],
|
||||
"Content-Length": str(length),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"ETag": f'"{row["sha256"]}"',
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
}
|
||||
return chunks, headers
|
||||
|
||||
|
||||
def _in_background(compensation) -> None:
|
||||
"""Do not keep the client waiting for compensation: with S3 down each
|
||||
attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503
|
||||
anyway."""
|
||||
task = asyncio.create_task(compensation)
|
||||
_compensations.add(task)
|
||||
task.add_done_callback(_compensations.discard)
|
||||
|
||||
|
||||
# --- Janitor ----------------------------------------------------------------
|
||||
|
||||
async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int:
|
||||
"""Delete objects of upload intents older than the grace period.
|
||||
|
||||
One intent per transaction: lock it (SKIP LOCKED), delete the object in
|
||||
S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and
|
||||
stop: the intent stays and the next run retries.
|
||||
"""
|
||||
cleaned = 0
|
||||
for _ in range(limit):
|
||||
async with SessionLocal() as db:
|
||||
row = await repositories.lock_stale_intent(db, grace_seconds)
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
break
|
||||
|
||||
if not row["committed"]:
|
||||
try:
|
||||
await storage.delete(row["object_key"])
|
||||
except StorageUnavailable:
|
||||
await db.rollback()
|
||||
logger.warning("janitor: S3 unavailable, will retry")
|
||||
break
|
||||
|
||||
await repositories.delete_upload_intent(db, row["object_key"])
|
||||
cleaned += 1
|
||||
logger.warning(
|
||||
"janitor: removed abandoned upload %s (object %s)",
|
||||
row["object_key"],
|
||||
"kept, row committed" if row["committed"] else "deleted",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
async def janitor_loop() -> None:
|
||||
while True:
|
||||
# Jitter: replicas do not all wake up at the same moment.
|
||||
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
|
||||
try:
|
||||
await cleanup_stale_uploads(settings.upload_grace_seconds)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("janitor run failed: %s", type(exc).__name__)
|
||||
@@ -63,6 +63,39 @@ class Settings:
|
||||
# for in-flight requests up to shutdown_timeout_seconds.
|
||||
shutdown_drain_seconds: float
|
||||
shutdown_timeout_seconds: int
|
||||
# Lab4: attachments. Metadata lives in PostgreSQL, bytes in a private
|
||||
# S3 bucket; the API uses its own restricted credentials (not root).
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
s3_object_prefix: str
|
||||
s3_connect_timeout_seconds: float
|
||||
s3_read_timeout_seconds: float
|
||||
# Deadline of one whole S3 call (DNS + retries included), below the LB
|
||||
# "timeout server" (10 s): a stopped S3 container makes Docker DNS hang
|
||||
# ~8 s per lookup, which connect_timeout does not cover.
|
||||
s3_operation_timeout_seconds: float
|
||||
# PUT gets operation timeout + size / this rate: 10 MiB under load is
|
||||
# slower than a HEAD. HAProxy allows 40 s for an upload POST.
|
||||
s3_put_min_bytes_per_second: int
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
# One upload may not take unlimited RAM/time: the body is spooled to a
|
||||
# temporary file (only upload_spool_memory_bytes stay in RAM), receiving
|
||||
# it is limited by upload_timeout_seconds, and at most
|
||||
# max_concurrent_uploads uploads per replica run at once.
|
||||
upload_spool_memory_bytes: int
|
||||
upload_timeout_seconds: float
|
||||
max_concurrent_uploads: int
|
||||
upload_slot_wait_seconds: float
|
||||
# Janitor: an upload intent older than upload_grace_seconds means the
|
||||
# request died between S3 PUT and COMMIT; its object is deleted.
|
||||
upload_grace_seconds: int
|
||||
janitor_interval_seconds: float
|
||||
# Experiments only (see REPORT.md): "" | crash_after_s3_put | db_error_after_s3_put
|
||||
fault_injection: str
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
@@ -106,6 +139,41 @@ def load_settings() -> Settings:
|
||||
shutdown_timeout_seconds=read_positive_int(
|
||||
"SHUTDOWN_TIMEOUT_SECONDS", "8"
|
||||
),
|
||||
s3_endpoint=read_value("S3_ENDPOINT", "http://s3:9000"),
|
||||
s3_region=os.getenv("S3_REGION", "us-east-1"),
|
||||
s3_bucket=read_value("S3_BUCKET", "messenger-attachments"),
|
||||
s3_access_key_id=read_value("S3_ACCESS_KEY_ID"),
|
||||
s3_secret_access_key=read_value("S3_SECRET_ACCESS_KEY"),
|
||||
s3_object_prefix=os.getenv("S3_OBJECT_PREFIX", "originals/"),
|
||||
s3_connect_timeout_seconds=read_positive_float(
|
||||
"S3_CONNECT_TIMEOUT_SECONDS", "2"
|
||||
),
|
||||
s3_read_timeout_seconds=read_positive_float(
|
||||
"S3_READ_TIMEOUT_SECONDS", "5"
|
||||
),
|
||||
s3_operation_timeout_seconds=read_positive_float(
|
||||
"S3_OPERATION_TIMEOUT_SECONDS", "6"
|
||||
),
|
||||
s3_put_min_bytes_per_second=read_positive_int(
|
||||
"S3_PUT_MIN_BYTES_PER_SECOND", "1048576"
|
||||
),
|
||||
max_upload_bytes=read_positive_int("MAX_UPLOAD_BYTES", "10485760"),
|
||||
max_image_pixels=read_positive_int("MAX_IMAGE_PIXELS", "40000000"),
|
||||
upload_spool_memory_bytes=read_positive_int(
|
||||
"UPLOAD_SPOOL_MEMORY_BYTES", "1048576"
|
||||
),
|
||||
upload_timeout_seconds=read_positive_float(
|
||||
"UPLOAD_TIMEOUT_SECONDS", "60"
|
||||
),
|
||||
max_concurrent_uploads=read_positive_int("MAX_CONCURRENT_UPLOADS", "8"),
|
||||
upload_slot_wait_seconds=read_positive_float(
|
||||
"UPLOAD_SLOT_WAIT_SECONDS", "10"
|
||||
),
|
||||
upload_grace_seconds=read_positive_int("UPLOAD_GRACE_SECONDS", "900"),
|
||||
janitor_interval_seconds=read_positive_float(
|
||||
"JANITOR_INTERVAL_SECONDS", "60"
|
||||
),
|
||||
fault_injection=os.getenv("FAULT_INJECTION", ""),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -55,6 +55,29 @@ class Unavailable(ApiError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(503, "unavailable", "Service is unavailable.")
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
def __init__(self, message: str = "Payload is too large.") -> None:
|
||||
super().__init__(413, "payload_too_large", message)
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
def __init__(
|
||||
self,
|
||||
message: str = "Only PNG and JPEG images are accepted.",
|
||||
) -> None:
|
||||
super().__init__(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
class RateLimited(ApiError):
|
||||
def __init__(self, message: str, retry_after_seconds: int = 1) -> None:
|
||||
super().__init__(
|
||||
429,
|
||||
"rate_limited",
|
||||
message,
|
||||
{"Retry-After": str(retry_after_seconds)},
|
||||
)
|
||||
|
||||
# X-Request-Id from HAProxy (or the client) is reused only if it is short and
|
||||
# printable; anything else is replaced, so it cannot inject into logs.
|
||||
REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
+264
@@ -0,0 +1,264 @@
|
||||
"""Content-based validation of uploaded PNG/JPEG images.
|
||||
|
||||
The declared Content-Type is never trusted: the format is detected from the
|
||||
first bytes and must match it. Dimensions are read from the header and the
|
||||
pixel limit is applied BEFORE any decoding, so a small "pixel bomb" file
|
||||
cannot make the API allocate gigabytes.
|
||||
|
||||
- PNG is validated by our own streaming reader: chunk lengths, CRC of every
|
||||
chunk, IHDR, and the IDAT zlib stream is decompressed in 64 KiB pieces
|
||||
that are counted and thrown away. Memory stays O(64 KiB) for any image.
|
||||
- JPEG: Pillow reads the header (lazy open); then the image is decoded with
|
||||
draft() at 1/8 scale, which makes libjpeg read the whole entropy-coded
|
||||
stream (a truncated/corrupted file raises) while allocating only 1/64 of
|
||||
the full-size pixel buffer. Large progressive JPEGs keep all DCT
|
||||
coefficients in memory even with draft(), so above
|
||||
PROGRESSIVE_DECODE_MAX_PIXELS only the marker structure is checked.
|
||||
|
||||
The functions are synchronous (CPU work); call them via asyncio.to_thread.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
# Our own limit is applied before Pillow decodes anything.
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SIGNATURE = b"\xff\xd8\xff"
|
||||
CHUNK = 64 * 1024
|
||||
PROGRESSIVE_DECODE_MAX_PIXELS = 16_000_000
|
||||
|
||||
|
||||
class InvalidImage(Exception):
|
||||
"""Not a PNG/JPEG, a corrupted file or a Content-Type mismatch (-> 415)."""
|
||||
|
||||
|
||||
class TooManyPixels(Exception):
|
||||
"""Decoded image would exceed MAX_IMAGE_PIXELS (-> 413)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def sniff_content_type(head: bytes) -> str | None:
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head.startswith(JPEG_SIGNATURE):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def validate_image(
|
||||
file: BinaryIO,
|
||||
declared_content_type: str,
|
||||
max_pixels: int,
|
||||
) -> ImageInfo:
|
||||
file.seek(0)
|
||||
detected = sniff_content_type(file.read(len(PNG_SIGNATURE)))
|
||||
file.seek(0)
|
||||
|
||||
if detected is None:
|
||||
raise InvalidImage("Content is not a PNG or JPEG image.")
|
||||
|
||||
if detected != declared_content_type:
|
||||
raise InvalidImage("Content-Type does not match the image content.")
|
||||
|
||||
if detected == "image/png":
|
||||
width, height = _validate_png(file, max_pixels)
|
||||
else:
|
||||
width, height = _validate_jpeg(file, max_pixels)
|
||||
|
||||
return ImageInfo(detected, width, height)
|
||||
|
||||
|
||||
# --- PNG -------------------------------------------------------------------
|
||||
|
||||
# color type -> (channels, allowed bit depths)
|
||||
_PNG_COLOR_TYPES = {
|
||||
0: (1, {1, 2, 4, 8, 16}),
|
||||
2: (3, {8, 16}),
|
||||
3: (1, {1, 2, 4, 8}),
|
||||
4: (2, {8, 16}),
|
||||
6: (4, {8, 16}),
|
||||
}
|
||||
_ADAM7_PASSES = (
|
||||
(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4),
|
||||
(0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2),
|
||||
)
|
||||
|
||||
|
||||
def _read_exact(file: BinaryIO, size: int) -> bytes:
|
||||
data = file.read(size)
|
||||
if len(data) != size:
|
||||
raise InvalidImage("Image file is truncated.")
|
||||
return data
|
||||
|
||||
|
||||
def _png_raw_size(width: int, height: int, bits_per_pixel: int, interlace: int) -> int:
|
||||
"""Size of the decompressed IDAT stream: filter byte + packed row bytes."""
|
||||
def scanlines(w: int, h: int) -> int:
|
||||
if w == 0 or h == 0:
|
||||
return 0
|
||||
return h * (1 + (w * bits_per_pixel + 7) // 8)
|
||||
|
||||
if interlace == 0:
|
||||
return scanlines(width, height)
|
||||
|
||||
total = 0
|
||||
for x0, y0, dx, dy in _ADAM7_PASSES:
|
||||
pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
total += scanlines(pass_width, pass_height)
|
||||
return total
|
||||
|
||||
|
||||
def _validate_png(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
_read_exact(file, len(PNG_SIGNATURE))
|
||||
|
||||
width = height = 0
|
||||
expected_raw = 0
|
||||
produced_raw = 0
|
||||
inflater = zlib.decompressobj()
|
||||
color_type = -1
|
||||
seen_plte = False
|
||||
idat_state = "before" # before -> inside -> after
|
||||
first = True
|
||||
|
||||
while True:
|
||||
header = _read_exact(file, 8)
|
||||
length, chunk_type = struct.unpack(">I4s", header)
|
||||
|
||||
if length > 0x7FFFFFFF:
|
||||
raise InvalidImage("Invalid PNG chunk length.")
|
||||
|
||||
if first and chunk_type != b"IHDR":
|
||||
raise InvalidImage("PNG must start with IHDR.")
|
||||
|
||||
crc = zlib.crc32(chunk_type)
|
||||
is_idat = chunk_type == b"IDAT"
|
||||
|
||||
if is_idat:
|
||||
if idat_state == "after":
|
||||
raise InvalidImage("PNG IDAT chunks are not consecutive.")
|
||||
if color_type == 3 and not seen_plte:
|
||||
raise InvalidImage("PNG palette image without PLTE.")
|
||||
idat_state = "inside"
|
||||
elif idat_state == "inside":
|
||||
idat_state = "after"
|
||||
|
||||
data = b""
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = _read_exact(file, min(CHUNK, remaining))
|
||||
remaining -= len(piece)
|
||||
crc = zlib.crc32(piece, crc)
|
||||
|
||||
if is_idat:
|
||||
produced_raw += _inflate_counting(
|
||||
inflater, piece, expected_raw - produced_raw
|
||||
)
|
||||
elif chunk_type == b"IHDR":
|
||||
data += piece
|
||||
|
||||
(stored_crc,) = struct.unpack(">I", _read_exact(file, 4))
|
||||
if stored_crc != crc:
|
||||
raise InvalidImage("PNG chunk CRC mismatch.")
|
||||
|
||||
if first:
|
||||
first = False
|
||||
if length != 13:
|
||||
raise InvalidImage("Invalid PNG IHDR.")
|
||||
(width, height, bit_depth, color_type,
|
||||
compression, filter_method, interlace) = struct.unpack(">IIBBBBB", data)
|
||||
|
||||
if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF:
|
||||
raise InvalidImage("Invalid PNG dimensions.")
|
||||
if width * height > max_pixels:
|
||||
# Before a single IDAT byte is decompressed.
|
||||
raise TooManyPixels()
|
||||
if color_type not in _PNG_COLOR_TYPES:
|
||||
raise InvalidImage("Invalid PNG color type.")
|
||||
channels, depths = _PNG_COLOR_TYPES[color_type]
|
||||
if bit_depth not in depths or compression != 0 or filter_method != 0 \
|
||||
or interlace not in (0, 1):
|
||||
raise InvalidImage("Invalid PNG IHDR.")
|
||||
expected_raw = _png_raw_size(width, height, channels * bit_depth, interlace)
|
||||
elif chunk_type == b"PLTE":
|
||||
seen_plte = True
|
||||
elif chunk_type == b"IEND":
|
||||
break
|
||||
|
||||
if idat_state == "before":
|
||||
raise InvalidImage("PNG has no image data.")
|
||||
|
||||
produced_raw += len(inflater.flush())
|
||||
if not inflater.eof or produced_raw != expected_raw:
|
||||
raise InvalidImage("PNG image data is corrupted or truncated.")
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
def _inflate_counting(inflater, data: bytes, allowed: int) -> int:
|
||||
"""Decompress and discard; return the number of produced bytes."""
|
||||
produced = 0
|
||||
try:
|
||||
while data:
|
||||
if inflater.eof:
|
||||
raise InvalidImage("PNG has data after the end of the zlib stream.")
|
||||
out = inflater.decompress(data, CHUNK)
|
||||
produced += len(out)
|
||||
if produced > allowed:
|
||||
raise InvalidImage("PNG image data is larger than IHDR declares.")
|
||||
data = inflater.unconsumed_tail
|
||||
except zlib.error:
|
||||
raise InvalidImage("PNG image data is corrupted.") from None
|
||||
return produced
|
||||
|
||||
|
||||
# --- JPEG ------------------------------------------------------------------
|
||||
|
||||
def _validate_jpeg(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
try:
|
||||
with Image.open(file, formats=["JPEG"]) as image:
|
||||
width, height = image.size
|
||||
|
||||
if width < 1 or height < 1:
|
||||
raise InvalidImage("Invalid JPEG dimensions.")
|
||||
if width * height > max_pixels:
|
||||
# Header only: no pixel has been decoded yet.
|
||||
raise TooManyPixels()
|
||||
|
||||
progressive = bool(
|
||||
image.info.get("progressive") or image.info.get("progression")
|
||||
)
|
||||
|
||||
if progressive and width * height > PROGRESSIVE_DECODE_MAX_PIXELS:
|
||||
_check_jpeg_end(file)
|
||||
else:
|
||||
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
image.load()
|
||||
except (InvalidImage, TooManyPixels):
|
||||
raise
|
||||
except Exception:
|
||||
# Pillow raises OSError/SyntaxError/ValueError/... for broken data.
|
||||
raise InvalidImage("JPEG image is corrupted or truncated.") from None
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
def _check_jpeg_end(file: BinaryIO) -> None:
|
||||
file.seek(0, 2)
|
||||
size = file.tell()
|
||||
file.seek(max(0, size - 2))
|
||||
if file.read(2) != b"\xff\xd9":
|
||||
raise InvalidImage("JPEG image is truncated.")
|
||||
+103
-5
@@ -1,17 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, Request, Response, Security
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, Query, Request, Response, Security
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from sqlalchemy import exc as sa_exc
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.attachments import (
|
||||
get_attachment,
|
||||
janitor_loop,
|
||||
open_attachment_content,
|
||||
upload_attachment,
|
||||
)
|
||||
from app.cache import session_cache
|
||||
from app.db import check_database_ready, engine, get_db
|
||||
from app.deps import basic_scheme, pagination_params, require_user
|
||||
@@ -25,6 +34,7 @@ from app.errors import (
|
||||
)
|
||||
from app.lifecycle import draining
|
||||
from app.models import (
|
||||
Attachment,
|
||||
Chat,
|
||||
ChatPage,
|
||||
CreateChat,
|
||||
@@ -36,6 +46,7 @@ from app.models import (
|
||||
Session,
|
||||
User,
|
||||
)
|
||||
from app.storage import storage
|
||||
from app.services import (
|
||||
create_chat,
|
||||
create_message,
|
||||
@@ -50,14 +61,19 @@ from app.services import (
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||
await storage.start()
|
||||
janitor = asyncio.create_task(janitor_loop())
|
||||
yield
|
||||
janitor.cancel()
|
||||
await asyncio.gather(janitor, return_exceptions=True)
|
||||
await storage.close()
|
||||
await session_cache.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUSI Messenger — лабораторная 3",
|
||||
version="1.3.0",
|
||||
title="MTUSI Messenger — лабораторная 4",
|
||||
version="1.4.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@@ -185,6 +201,9 @@ async def liveness() -> dict[str, str]:
|
||||
async def readiness() -> dict[str, str]:
|
||||
# A draining replica reports "not ready" so HAProxy stops sending new
|
||||
# requests to it before the listener is closed.
|
||||
# S3 is deliberately NOT checked: without it only attachments degrade
|
||||
# (503), users/chats/messages keep working. If readiness depended on S3,
|
||||
# an S3 outage would take every replica out of the LB.
|
||||
if draining.is_set() or not await check_database_ready():
|
||||
raise Unavailable()
|
||||
|
||||
@@ -331,6 +350,7 @@ async def create_message_route(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
text_value=payload.text,
|
||||
attachment_ids=payload.attachment_ids,
|
||||
idempotency_key=idempotency_key,
|
||||
current_user=current_user,
|
||||
)
|
||||
@@ -359,4 +379,82 @@ async def list_messages_route(
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
current_user=current_user,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
UPLOAD_BODY = {
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"description": "Raw image bytes, not multipart and not JSON/base64.",
|
||||
"content": {
|
||||
"image/png": {"schema": {"type": "string", "format": "binary"}},
|
||||
"image/jpeg": {"schema": {"type": "string", "format": "binary"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
status_code=201,
|
||||
response_model=Attachment,
|
||||
openapi_extra=UPLOAD_BODY,
|
||||
)
|
||||
async def upload_attachment_route(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
filename: str = Query(min_length=1, max_length=255),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await upload_attachment(
|
||||
request,
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
filename=filename,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
response_model=Attachment,
|
||||
)
|
||||
async def get_attachment_route(
|
||||
attachment_id: UUID,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await get_attachment(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/attachments/{attachment_id}/content",
|
||||
response_class=StreamingResponse,
|
||||
responses={200: {"content": {"image/png": {}, "image/jpeg": {}}}},
|
||||
)
|
||||
async def download_attachment_route(
|
||||
attachment_id: UUID,
|
||||
# Known variant names are accepted so that a ready attachment without
|
||||
# that variant answers 404 (contract); any other value is 400.
|
||||
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(
|
||||
default="original"
|
||||
),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
chunks, headers = await open_attachment_content(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
variant=variant,
|
||||
current_user=current_user,
|
||||
)
|
||||
return StreamingResponse(
|
||||
chunks,
|
||||
media_type=headers.pop("Content-Type"),
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
+50
-6
@@ -4,7 +4,7 @@ from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class ApiModel(BaseModel):
|
||||
@@ -92,15 +92,26 @@ class ChatPage(ApiModel):
|
||||
|
||||
|
||||
class CreateMessage(ApiModel):
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
# Lab4: text may be "" when the message carries attachments.
|
||||
text: str = Field(max_length=4000)
|
||||
# 0-10 unique IDs of attachments of the same chat; missing == [].
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("text")
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def text_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("text must not contain only spaces.")
|
||||
def attachment_ids_are_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("attachment_ids must be unique.")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def text_or_attachments(self) -> "CreateMessage":
|
||||
if not self.attachment_ids and not self.text.strip():
|
||||
raise ValueError(
|
||||
"text must not be blank when there are no attachments."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class Message(ApiModel):
|
||||
id: UUID
|
||||
@@ -108,6 +119,7 @@ class Message(ApiModel):
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[UUID] = Field(default_factory=list)
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
@@ -136,6 +148,38 @@ class Session(ApiModel):
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Variant(ApiModel):
|
||||
name: Literal["thumbnail", "crop", "optimized"]
|
||||
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(ApiModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: Literal["image/png", "image/jpeg"]
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
# Lab4: synchronous upload, a stored attachment is always ready.
|
||||
status: Literal["ready"]
|
||||
variants: list[Variant]
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@field_validator("created_at", "updated_at")
|
||||
@classmethod
|
||||
def attachment_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("timestamps must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Health(ApiModel):
|
||||
status: Literal["ok"]
|
||||
|
||||
|
||||
+270
-13
@@ -1,4 +1,4 @@
|
||||
"""PostgreSQL repository functions for MTUSI Messenger Lab3."""
|
||||
"""PostgreSQL repository functions for MTUSI Messenger Lab4."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
@@ -301,12 +301,28 @@ async def list_chats_for_user(
|
||||
return chats
|
||||
|
||||
|
||||
# Attachment IDs of a message in the order of the request.
|
||||
MESSAGE_ATTACHMENT_IDS = """
|
||||
COALESCE(
|
||||
(SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FROM message_attachments AS ma
|
||||
WHERE ma.message_id = m.id),
|
||||
CAST('{}' AS uuid[])
|
||||
) AS attachment_ids
|
||||
"""
|
||||
|
||||
|
||||
class AttachmentsNotFound(Exception):
|
||||
"""Some attachment_ids do not exist or belong to another chat (-> 404)."""
|
||||
|
||||
|
||||
async def create_message_idempotent(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text_value: str,
|
||||
attachment_ids: list[UUID],
|
||||
idempotency_key: str,
|
||||
request_hash: str,
|
||||
ttl_seconds: int,
|
||||
@@ -316,10 +332,11 @@ async def create_message_idempotent(
|
||||
1. INSERT the key row. Its primary key (user, chat, method, key) lives in
|
||||
the shared PostgreSQL, so a concurrent request with the same key on
|
||||
ANY replica blocks on this INSERT until the first transaction ends.
|
||||
2. If the key was claimed: INSERT the message, link it to the key and
|
||||
COMMIT both together. A failure before COMMIT rolls back both, so a
|
||||
transient error is never stored as the final answer and a retry with
|
||||
the same key simply runs again.
|
||||
2. If the key was claimed: check that every attachment belongs to this
|
||||
chat, INSERT the message and its attachment links, link the message
|
||||
to the key and COMMIT everything together. A failure before COMMIT
|
||||
(also AttachmentsNotFound) rolls back all of it, so an error is never
|
||||
stored as the final answer and a retry with the same key runs again.
|
||||
3. If the key already exists (committed by someone else): return the
|
||||
stored message, or None when the body differs (-> 409).
|
||||
|
||||
@@ -351,6 +368,23 @@ async def create_message_idempotent(
|
||||
)
|
||||
|
||||
if claimed.first() is not None:
|
||||
if attachment_ids:
|
||||
# Unknown and foreign IDs look the same (404): the response
|
||||
# does not reveal that an attachment of another chat exists.
|
||||
found = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id FROM attachments
|
||||
WHERE chat_id = :chat_id
|
||||
AND id = ANY(CAST(:ids AS uuid[]))
|
||||
FOR SHARE
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id, "ids": attachment_ids},
|
||||
)
|
||||
if {row.id for row in found} != set(attachment_ids):
|
||||
raise AttachmentsNotFound()
|
||||
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -368,6 +402,25 @@ async def create_message_idempotent(
|
||||
)
|
||||
message = dict(result.mappings().one())
|
||||
|
||||
if attachment_ids:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO message_attachments
|
||||
(message_id, attachment_id, position)
|
||||
VALUES (:message_id, :attachment_id, :position)
|
||||
"""
|
||||
),
|
||||
[
|
||||
{
|
||||
"message_id": message["id"],
|
||||
"attachment_id": attachment_id,
|
||||
"position": position,
|
||||
}
|
||||
for position, attachment_id in enumerate(attachment_ids)
|
||||
],
|
||||
)
|
||||
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -382,13 +435,15 @@ async def create_message_idempotent(
|
||||
{**key_params, "message_id": message["id"]},
|
||||
)
|
||||
await db.commit()
|
||||
message["attachment_ids"] = list(attachment_ids)
|
||||
return message, False
|
||||
|
||||
stored = await db.execute(
|
||||
text(
|
||||
"""
|
||||
f"""
|
||||
SELECT k.request_hash,
|
||||
m.id, m.chat_id, m.sender_id, m.text, m.created_at
|
||||
m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{MESSAGE_ATTACHMENT_IDS}
|
||||
FROM idempotency_keys AS k
|
||||
JOIN messages AS m ON m.id = k.message_id
|
||||
WHERE k.user_id = :user_id
|
||||
@@ -424,10 +479,11 @@ async def list_messages_keyset(
|
||||
cursor_created_at: datetime | None,
|
||||
cursor_id: UUID | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = """
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :chat_id \
|
||||
query = f"""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{MESSAGE_ATTACHMENT_IDS}
|
||||
FROM messages AS m
|
||||
WHERE m.chat_id = :chat_id
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"chat_id": chat_id,
|
||||
@@ -436,11 +492,212 @@ async def list_messages_keyset(
|
||||
|
||||
if cursor_created_at is not None and cursor_id is not None:
|
||||
query += """
|
||||
AND (created_at, id) > (:cursor_created_at, :cursor_id)
|
||||
AND (m.created_at, m.id) > (:cursor_created_at, :cursor_id)
|
||||
"""
|
||||
params["cursor_created_at"] = cursor_created_at
|
||||
params["cursor_id"] = cursor_id
|
||||
|
||||
query += " ORDER BY created_at ASC, id ASC LIMIT :limit_plus_one"
|
||||
query += " ORDER BY m.created_at ASC, m.id ASC LIMIT :limit_plus_one"
|
||||
result = await db.execute(text(query), params)
|
||||
return [dict(row) for row in result.mappings().all()]
|
||||
|
||||
|
||||
# --- Attachments ------------------------------------------------------------
|
||||
|
||||
ATTACHMENT_COLUMNS = """
|
||||
a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
|
||||
a.sha256, a.status, a.error_code, a.object_key, a.created_at, a.updated_at
|
||||
"""
|
||||
|
||||
|
||||
async def create_upload_intent(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
object_key: str,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
) -> None:
|
||||
"""Committed BEFORE the S3 PUT: from now on the object can be found and
|
||||
cleaned up even if this process dies right after the PUT."""
|
||||
try:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO upload_intents (object_key, attachment_id, chat_id)
|
||||
VALUES (:object_key, :attachment_id, :chat_id)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"object_key": object_key,
|
||||
"attachment_id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def finalize_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
object_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Delete the intent and insert the ready attachment in ONE transaction.
|
||||
|
||||
Returns None if the intent is already gone: the janitor decided this
|
||||
upload was abandoned and deleted the object, so the row must not be
|
||||
created. The DELETE waits on the janitor's row lock, so the two can
|
||||
never both succeed.
|
||||
"""
|
||||
try:
|
||||
claimed = await db.execute(
|
||||
text(
|
||||
"DELETE FROM upload_intents WHERE object_key = :object_key RETURNING 1"
|
||||
),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
if claimed.first() is None:
|
||||
await db.rollback()
|
||||
return None
|
||||
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
INSERT INTO attachments AS a
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, width, height, object_key)
|
||||
VALUES (:id, :chat_id, :owner_id, :filename, :content_type,
|
||||
:size_bytes, :sha256, :width, :height, :object_key)
|
||||
RETURNING {ATTACHMENT_COLUMNS}
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
"owner_id": owner_id,
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size_bytes": size_bytes,
|
||||
"sha256": sha256,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"object_key": object_key,
|
||||
},
|
||||
)
|
||||
row = dict(result.mappings().one())
|
||||
await db.commit()
|
||||
return row
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def delete_upload_intent(db: AsyncSession, object_key: str) -> None:
|
||||
try:
|
||||
await db.execute(
|
||||
text("DELETE FROM upload_intents WHERE object_key = :object_key"),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def get_accessible_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Only members of the attachment's chat see it; others get None (404)."""
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT {ATTACHMENT_COLUMNS}
|
||||
FROM attachments AS a
|
||||
JOIN chat_members AS cm
|
||||
ON cm.chat_id = a.chat_id
|
||||
AND cm.user_id = :user_id
|
||||
WHERE a.id = :attachment_id
|
||||
"""
|
||||
),
|
||||
{"attachment_id": attachment_id, "user_id": user_id},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def lock_upload_intent(
|
||||
db: AsyncSession,
|
||||
object_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lock the intent of one upload (waits for a finishing transaction);
|
||||
the transaction stays open. None: no intent, nothing to compensate."""
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT i.object_key,
|
||||
EXISTS (SELECT 1 FROM attachments AS a
|
||||
WHERE a.object_key = i.object_key) AS committed
|
||||
FROM upload_intents AS i
|
||||
WHERE i.object_key = :object_key
|
||||
FOR UPDATE OF i
|
||||
"""
|
||||
),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def lock_stale_intent(
|
||||
db: AsyncSession,
|
||||
grace_seconds: int,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lock ONE intent older than the grace period; the transaction stays open.
|
||||
|
||||
SKIP LOCKED: an upload that is finishing right now holds the row lock in
|
||||
finalize_attachment and is skipped; several replicas can run the janitor
|
||||
at once without touching the same intent.
|
||||
"""
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT i.object_key,
|
||||
EXISTS (SELECT 1 FROM attachments AS a
|
||||
WHERE a.object_key = i.object_key) AS committed
|
||||
FROM upload_intents AS i
|
||||
WHERE i.created_at < now() - make_interval(secs => :grace)
|
||||
ORDER BY i.created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE OF i SKIP LOCKED
|
||||
"""
|
||||
),
|
||||
{"grace": grace_seconds},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def attachment_keys(db: AsyncSession) -> set[str]:
|
||||
result = await db.execute(text("SELECT object_key FROM attachments"))
|
||||
return {row.object_key for row in result}
|
||||
|
||||
|
||||
async def intent_keys(db: AsyncSession) -> dict[str, datetime]:
|
||||
result = await db.execute(
|
||||
text("SELECT object_key, created_at FROM upload_intents")
|
||||
)
|
||||
return {row.object_key: row.created_at for row in result}
|
||||
|
||||
+24
-13
@@ -1,4 +1,4 @@
|
||||
"""Business rules for MTUSI Messenger Lab3."""
|
||||
"""Business rules for MTUSI Messenger Lab4 (users, chats, messages)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
@@ -58,6 +58,7 @@ def _to_message(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"sender_id": row["sender_id"],
|
||||
"text": row["text"],
|
||||
"created_at": row["created_at"],
|
||||
"attachment_ids": list(row.get("attachment_ids") or []),
|
||||
}
|
||||
|
||||
|
||||
@@ -299,10 +300,15 @@ async def list_chats(
|
||||
}
|
||||
|
||||
|
||||
def _request_fingerprint(text_value: str) -> str:
|
||||
"""Hash of the validated body: key order and whitespace do not matter."""
|
||||
def _request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str:
|
||||
"""Hash of the validated body: key order and whitespace do not matter,
|
||||
a missing attachment_ids equals []. Without attachments the canonical
|
||||
form is the lab3 one, so keys stored by lab3 still replay correctly."""
|
||||
body: dict[str, Any] = {"text": text_value}
|
||||
if attachment_ids:
|
||||
body["attachment_ids"] = [str(item) for item in attachment_ids]
|
||||
canonical = json.dumps(
|
||||
{"text": text_value},
|
||||
body,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
@@ -315,6 +321,7 @@ async def create_message(
|
||||
*,
|
||||
chat_id: UUID,
|
||||
text_value: str,
|
||||
attachment_ids: list[UUID],
|
||||
idempotency_key: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
@@ -339,15 +346,19 @@ async def create_message(
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
result = await repositories.create_message_idempotent(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
sender_id=current_user["id"],
|
||||
text_value=text_value,
|
||||
idempotency_key=idempotency_key,
|
||||
request_hash=_request_fingerprint(text_value),
|
||||
ttl_seconds=settings.idempotency_ttl_seconds,
|
||||
)
|
||||
try:
|
||||
result = await repositories.create_message_idempotent(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
sender_id=current_user["id"],
|
||||
text_value=text_value,
|
||||
attachment_ids=attachment_ids,
|
||||
idempotency_key=idempotency_key,
|
||||
request_hash=_request_fingerprint(text_value, attachment_ids),
|
||||
ttl_seconds=settings.idempotency_ttl_seconds,
|
||||
)
|
||||
except repositories.AttachmentsNotFound:
|
||||
raise _not_found() from None
|
||||
|
||||
if result is None:
|
||||
raise ApiError(
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway).
|
||||
|
||||
One long-lived aiobotocore client per API process, path-style addressing,
|
||||
short connect/read timeouts, at most 2 attempts and an overall deadline
|
||||
S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a
|
||||
fast 503 on the attachment endpoints only. Any client-side or
|
||||
network error is raised as StorageUnavailable; a missing key is
|
||||
ObjectMissing. Object keys are generated by the server (see services.py);
|
||||
the user's filename never becomes part of a key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import AsyncExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.storage")
|
||||
|
||||
DOWNLOAD_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 is unreachable, timed out or refused the request."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""The key does not exist in the bucket."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoredObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: Any
|
||||
|
||||
|
||||
class ObjectStorage:
|
||||
def __init__(self) -> None:
|
||||
self._stack: AsyncExitStack | None = None
|
||||
self._client = None
|
||||
self.bucket = settings.s3_bucket
|
||||
|
||||
async def start(self) -> None:
|
||||
self._stack = AsyncExitStack()
|
||||
session = get_session()
|
||||
self._client = await self._stack.enter_async_context(
|
||||
session.create_client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key_id,
|
||||
aws_secret_access_key=settings.s3_secret_access_key,
|
||||
config=AioConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=settings.s3_connect_timeout_seconds,
|
||||
read_timeout=settings.s3_read_timeout_seconds,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
max_pool_connections=settings.max_concurrent_uploads * 2 + 4,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._stack is not None:
|
||||
await self._stack.aclose()
|
||||
self._stack = None
|
||||
self._client = None
|
||||
|
||||
@staticmethod
|
||||
def _deadline(size: int = 0):
|
||||
return asyncio.timeout(
|
||||
settings.s3_operation_timeout_seconds
|
||||
+ size / settings.s3_put_min_bytes_per_second
|
||||
)
|
||||
|
||||
def _require_client(self):
|
||||
if self._client is None:
|
||||
raise StorageUnavailable("S3 client is not started")
|
||||
return self._client
|
||||
|
||||
async def put(
|
||||
self,
|
||||
key: str,
|
||||
body: BinaryIO,
|
||||
size: int,
|
||||
content_type: str,
|
||||
) -> None:
|
||||
"""Upload a seekable file. SigV4 signs the payload SHA-256, so S3
|
||||
itself rejects bytes that were damaged on the way."""
|
||||
client = self._require_client()
|
||||
body.seek(0)
|
||||
try:
|
||||
async with self._deadline(size):
|
||||
await client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=body,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]:
|
||||
"""GET the object; S3 errors surface here, before any byte is sent."""
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
response = await client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
raise ObjectMissing(key) from exc
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
body = response["Body"]
|
||||
|
||||
async def chunks() -> AsyncIterator[bytes]:
|
||||
try:
|
||||
async for chunk in body.iter_chunks(DOWNLOAD_CHUNK):
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return chunks(), int(response["ContentLength"])
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
"""Idempotent: deleting a missing key succeeds in S3."""
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
await client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
await client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return False
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]:
|
||||
client = self._require_client()
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
|
||||
for item in page.get("Contents", []):
|
||||
yield StoredObject(item["Key"], item["Size"], item["LastModified"])
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
|
||||
storage = ObjectStorage()
|
||||
@@ -1,12 +0,0 @@
|
||||
# Topology worksheet, deliberately NOT a runnable stack (no application supplied).
|
||||
# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment.
|
||||
# Service names below are logical roles, not imposed container names.
|
||||
x-lab-number: 4
|
||||
x-required-roles:
|
||||
- api
|
||||
- postgres
|
||||
- load_balancer
|
||||
- s3
|
||||
# TODO: implement real images/builds, network wiring, volumes, healthchecks,
|
||||
# credentials and bootstrap. Details: contracts/infrastructure.json and README.md.
|
||||
services: {}
|
||||
+103
-9
@@ -1,8 +1,9 @@
|
||||
#file: noinspection ComposeUnknownValues
|
||||
# Lab3: client -> HAProxy -> API x N -> PostgreSQL (source of truth: data,
|
||||
# sessions, idempotency keys) + Valkey (shared session cache from lab2).
|
||||
# Lab4: client -> HAProxy -> API x N -> PostgreSQL (source of truth: data,
|
||||
# sessions, idempotency keys, attachment metadata) + Valkey (session cache)
|
||||
# + SeaweedFS S3 (private bucket with attachment bytes).
|
||||
# Start: see REPORT.md, "Запуск из чистого clone".
|
||||
name: messenger-lab3
|
||||
name: messenger-lab4
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
@@ -23,6 +24,27 @@ x-api-env: &api-env
|
||||
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
|
||||
SHUTDOWN_DRAIN_SECONDS: "3"
|
||||
SHUTDOWN_TIMEOUT_SECONDS: "8"
|
||||
# Lab4: S3 with the API's own restricted credentials (not admin).
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_OBJECT_PREFIX: originals/
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_CONNECT_TIMEOUT_SECONDS: "2"
|
||||
S3_READ_TIMEOUT_SECONDS: "5"
|
||||
S3_OPERATION_TIMEOUT_SECONDS: "6"
|
||||
S3_PUT_MIN_BYTES_PER_SECOND: "1048576"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
UPLOAD_SPOOL_MEMORY_BYTES: "1048576"
|
||||
UPLOAD_TIMEOUT_SECONDS: "60"
|
||||
MAX_CONCURRENT_UPLOADS: "8"
|
||||
UPLOAD_SLOT_WAIT_SECONDS: "10"
|
||||
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
|
||||
JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}"
|
||||
# Experiments only: crash_after_s3_put | db_error_after_s3_put (REPORT.md).
|
||||
FAULT_INJECTION: "${FAULT_INJECTION:-}"
|
||||
|
||||
services:
|
||||
db:
|
||||
@@ -76,11 +98,71 @@ services:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
|
||||
# S3: SeaweedFS all-in-one. No "ports": reachable only from the internal
|
||||
# backend network; its admin web UIs are bound to 127.0.0.1 inside the
|
||||
# container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.97
|
||||
entrypoint: ["/bin/sh", "/opt/lab/start.sh"]
|
||||
environment:
|
||||
# Soft limit of the Go heap: without it the GC lets SeaweedFS grow to
|
||||
# the container limit under parallel 10 MiB PUTs (measured, REPORT.md).
|
||||
GOMEMLIMIT: 600MiB
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./infra/seaweedfs:/opt/lab:ro
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 60s
|
||||
networks: [backend]
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
# 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed
|
||||
# SeaweedFS to the limit and the kernel killed it (REPORT.md).
|
||||
limits: { cpus: "1.0", memory: 1G }
|
||||
|
||||
# Idempotent bucket bootstrap + self-checks (private bucket, least
|
||||
# privilege of the API key). The only container with the S3 admin key.
|
||||
s3-init:
|
||||
build: .
|
||||
image: messenger-lab4-api:${APP_VERSION:-dev}
|
||||
command: ["python", "scripts/s3_bootstrap.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_OBJECT_PREFIX: originals/
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3: { condition: service_healthy }
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
# Runs the migrations once, before any API replica starts (no race of
|
||||
# replicas). A repeated run only prints SKIP.
|
||||
migrate:
|
||||
build: .
|
||||
image: messenger-lab3-api:${APP_VERSION:-dev}
|
||||
image: messenger-lab4-api:${APP_VERSION:-dev}
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
@@ -95,9 +177,9 @@ services:
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: messenger-lab3-api:${APP_VERSION:-dev}
|
||||
image: messenger-lab4-api:${APP_VERSION:-dev}
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url]
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
# No "ports": replicas are reachable only through the LB.
|
||||
# No "container_name": the service is scaled by deploy.replicas / --scale.
|
||||
networks: [edge, backend]
|
||||
@@ -105,6 +187,7 @@ services:
|
||||
db: { condition: service_healthy }
|
||||
valkey: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
@@ -157,20 +240,23 @@ services:
|
||||
working_dir: /work
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
COMPOSE_PROJECT: messenger-lab3
|
||||
COMPOSE_PROJECT: messenger-lab4
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
volumes:
|
||||
- ./:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
networks: [edge]
|
||||
# backend: only for the "private bucket from outside the API" check.
|
||||
networks: [edge, backend]
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
# PostgreSQL and Valkey: no route outside, the LB is not attached.
|
||||
# PostgreSQL, Valkey and S3: no route outside, the LB is not attached.
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data: {}
|
||||
s3_data: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
@@ -181,3 +267,11 @@ secrets:
|
||||
file: ./infra/secrets/valkey_password.txt
|
||||
cache_url:
|
||||
file: ./infra/secrets/cache_url.txt
|
||||
s3_admin_access_key_id:
|
||||
file: ./infra/secrets/s3_admin_access_key_id.txt
|
||||
s3_admin_secret_access_key:
|
||||
file: ./infra/secrets/s3_admin_secret_access_key.txt
|
||||
s3_access_key_id:
|
||||
file: ./infra/secrets/s3_access_key_id.txt
|
||||
s3_secret_access_key:
|
||||
file: ./infra/secrets/s3_secret_access_key.txt
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
## f00b790a44e9
|
||||
mounts=[{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_access_key_id.txt","Destination":"/run/secrets/s3_access_key_id","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_secret_access_key.txt","Destination":"/run/secrets/s3_secret_access_key","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\database_url.txt","Destination":"/run/secrets/database_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\cache_url.txt","Destination":"/run/secrets/cache_url","Mode":"","RW":false,"Propagation":"rprivate"}] readonly_rootfs=false
|
||||
tmp files:
|
||||
total 8
|
||||
drwxrwxrwt 1 root root 4096 Oct 30 11:44 .
|
||||
drwxr-xr-x 1 root root 4096 Oct 30 11:44 ..
|
||||
files > 10KB written by appuser outside /proc,/sys:
|
||||
## bb5edb449d7c
|
||||
mounts=[{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\database_url.txt","Destination":"/run/secrets/database_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\cache_url.txt","Destination":"/run/secrets/cache_url","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_access_key_id.txt","Destination":"/run/secrets/s3_access_key_id","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"C:\\Users\\masha\\WebstormProjects\\mtusi2026lab4\\infra\\secrets\\s3_secret_access_key.txt","Destination":"/run/secrets/s3_secret_access_key","Mode":"","RW":false,"Propagation":"rprivate"}] readonly_rootfs=false
|
||||
tmp files:
|
||||
total 8
|
||||
drwxrwxrwt 2 root root 4096 Dec 2 2024 .
|
||||
drwxr-xr-x 1 root root 4096 Oct 30 11:44 ..
|
||||
files > 10KB written by appuser outside /proc,/sys:
|
||||
@@ -0,0 +1,63 @@
|
||||
Container messenger-lab4-tester-run-cb4971c6a617 Creating
|
||||
Container messenger-lab4-tester-run-cb4971c6a617 Created
|
||||
OK JPEG fixture -> 201 ready: 201 (ожидается 201 )
|
||||
OK участник скачивает JPEG побайтно, без redirect: 200 (ожидается 200 )
|
||||
OK PNG ровно 10 MiB -> 201: 201 (ожидается 201 )
|
||||
OK 10 MiB скачан побайтно: 200 (ожидается 200 )
|
||||
OK PNG ровно 40 млн пикселей -> 201: 201 (ожидается 201 )
|
||||
OK 10 MiB + 1 байт (Content-Length) -> 413: 413 payload_too_large (ожидается 413 payload_too_large)
|
||||
OK 11 MiB chunked без Content-Length -> 413: 413 payload_too_large (ожидается 413 payload_too_large)
|
||||
OK PNG-«бомба» 50 млн пикселей (~10 KB файл) -> 413: 413 payload_too_large (ожидается 413 payload_too_large)
|
||||
OK PNG 2^31-1 x 2^31-1 в IHDR -> 413: 413 payload_too_large (ожидается 413 payload_too_large)
|
||||
OK пустое тело -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK PNG обрезан пополам -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK PNG с неверным CRC -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK PNG с мусором в zlib при верном CRC -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK PNG без IEND -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK JPEG обрезан -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK JPEG только сигнатура -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK GIF как image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK случайные байты как image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK HTML как image/jpeg -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK PNG под Content-Type image/jpeg -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK JPEG под Content-Type image/png -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK PNG под text/plain -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK PNG под image/gif -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK multipart/form-data -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK без Content-Type -> 415: 415 unsupported_media_type (ожидается 415 unsupported_media_type)
|
||||
OK filename ../../etc/passwd -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename C:\boot.png -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename NUL в имени -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename перевод строки -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename ESC-последовательность -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename bidi override (gnp.exe) -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename только пробелы -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename .. -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename 256 символов -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename пустой filename -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename нет filename -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK filename ровно 255 символов -> 201: 201 (ожидается 201 )
|
||||
OK upload без токена -> 401: 401 unauthorized (ожидается 401 unauthorized)
|
||||
OK посторонний upload в чужой чат -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK upload в несуществующий чат -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK посторонний GET /attachments/id -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK неизвестный attachment -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK посторонний GET /attachments/id/content -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK неизвестный attachment/content -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK attachment_id не UUID -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK ready без варианта thumbnail -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK variant неизвестного имени -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK вложение другого чата (своего) -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK вложение чужого чата -> 404: 404 not_found (ожидается 404 not_found)
|
||||
OK одно из ID неизвестно -> 404, ничего не создано: 404 not_found (ожидается 404 not_found)
|
||||
OK дубли attachment_ids -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK 11 attachment_ids -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK attachment_id не UUID -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK пробельный text без вложений -> 400: 400 invalid_request (ожидается 400 invalid_request)
|
||||
OK Bob: text="" + вложение Alice из общего чата -> 201: 201 (ожидается 201 )
|
||||
OK повтор Idempotency-Key -> то же сообщение: 201 (ожидается 201 )
|
||||
OK тот же ключ, другое тело -> 409: 409 conflict (ожидается 409 conflict)
|
||||
OK история: ровно одно сообщение с вложением: 200 (ожидается 200 )
|
||||
OK после оборванной загрузки API работает: 201 (ожидается 201 )
|
||||
OK ошибка в общем формате с request_id: 415 unsupported_media_type (ожидается 415 )
|
||||
PASS: граничные случаи вложений соответствуют контракту
|
||||
@@ -0,0 +1,30 @@
|
||||
## s3-init run 1
|
||||
keep bucket messenger-attachments
|
||||
PASS api credentials: PUT object
|
||||
PASS api credentials: GET object
|
||||
PASS anonymous GET object -> 403
|
||||
PASS anonymous LIST bucket -> 403
|
||||
PASS anonymous PUT object -> 403
|
||||
PASS api credentials: CreateBucket denied
|
||||
PASS api credentials: DeleteBucket denied
|
||||
PASS api credentials: DELETE object
|
||||
PASS api credentials: ListBuckets shows no foreign buckets
|
||||
PASS: bucket bootstrap complete (idempotent, private, least privilege)
|
||||
## s3-init run 2
|
||||
keep bucket messenger-attachments
|
||||
PASS api credentials: PUT object
|
||||
PASS api credentials: GET object
|
||||
PASS anonymous GET object -> 403
|
||||
PASS anonymous LIST bucket -> 403
|
||||
PASS anonymous PUT object -> 403
|
||||
PASS api credentials: CreateBucket denied
|
||||
PASS api credentials: DeleteBucket denied
|
||||
PASS api credentials: DELETE object
|
||||
PASS api credentials: ListBuckets shows no foreign buckets
|
||||
PASS: bucket bootstrap complete (idempotent, private, least privilege)
|
||||
## migrate rerun
|
||||
SKIP 001_initial_schema.sql
|
||||
SKIP 002_indexes.sql
|
||||
SKIP 003_idempotency_keys.sql
|
||||
SKIP 004_attachments.sql
|
||||
PASS: migrations applied
|
||||
@@ -0,0 +1,24 @@
|
||||
## run 1
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 3.972}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
messenger-lab4-api-2 bb5edb449d7c Up 41 seconds (healthy)
|
||||
messenger-lab4-api-1 f00b790a44e9 Exited (0) 12 seconds ago
|
||||
messenger-lab3-api-6 82d4e8380bcc Exited (0) 6 days ago
|
||||
messenger-lab3-api-5 9478eeefc0a7 Exited (0) 6 days ago
|
||||
mtusi2026lab2-api-1 01469cb1e76e Exited (137) 2 weeks ago
|
||||
started api container f00b790a44e9
|
||||
running api replicas: 2
|
||||
## run 2
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 4.208}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
messenger-lab4-api-2 bb5edb449d7c Up About a minute (healthy)
|
||||
messenger-lab4-api-1 f00b790a44e9 Exited (0) 11 seconds ago
|
||||
messenger-lab3-api-6 82d4e8380bcc Exited (0) 6 days ago
|
||||
messenger-lab3-api-5 9478eeefc0a7 Exited (0) 6 days ago
|
||||
mtusi2026lab2-api-1 01469cb1e76e Exited (137) 2 weeks ago
|
||||
started api container f00b790a44e9
|
||||
running api replicas: 2
|
||||
@@ -0,0 +1,22 @@
|
||||
## before
|
||||
messenger-lab4-api-1 f00b790a44e9 started=2026-10-30T11:44:20.074794065Z status=running health=healthy
|
||||
messenger-lab4-api-2 bb5edb449d7c started=2026-10-30T11:44:19.748733251Z status=running health=healthy
|
||||
messenger-lab4-db-1 331396b7fb10 started=2026-10-30T11:09:07.431846589Z status=running health=healthy
|
||||
messenger-lab4-lb-1 ffe0343f6dc2 started=2026-10-30T11:23:41.657935704Z status=running health=healthy
|
||||
messenger-lab4-s3-1 16bae2ec7298 started=2026-10-30T11:45:54.317998359Z status=running health=healthy
|
||||
messenger-lab4-valkey-1 b7f2d1a478a8 started=2026-10-30T11:09:07.702076066Z status=running health=healthy
|
||||
volume messenger-lab4_postgres_data
|
||||
volume messenger-lab4_s3_data
|
||||
## test
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
## after
|
||||
messenger-lab4-api-1 f00b790a44e9 started=2026-10-30T11:47:37.860335522Z status=running health=healthy
|
||||
messenger-lab4-api-2 bb5edb449d7c started=2026-10-30T11:47:37.593545231Z status=running health=healthy
|
||||
messenger-lab4-db-1 331396b7fb10 started=2026-10-30T11:47:11.170703341Z status=running health=healthy
|
||||
messenger-lab4-lb-1 ffe0343f6dc2 started=2026-10-30T11:47:44.439408787Z status=running health=starting
|
||||
messenger-lab4-s3-1 16bae2ec7298 started=2026-10-30T11:47:11.637306219Z status=running health=healthy
|
||||
messenger-lab4-valkey-1 b7f2d1a478a8 started=2026-10-30T11:47:11.434151225Z status=running health=healthy
|
||||
volume messenger-lab4_postgres_data
|
||||
volume messenger-lab4_s3_data
|
||||
@@ -0,0 +1,34 @@
|
||||
Container messenger-lab4-tester-run-cabae88c67e1 Creating
|
||||
Container messenger-lab4-tester-run-cabae88c67e1 Created
|
||||
OK object_key есть в PostgreSQL и не совпадает с filename/ID
|
||||
OK API не раскрывает object_key
|
||||
OK анонимный GET /messenger-attachments/<key> напрямую в S3 -> 403 (403)
|
||||
OK анонимный HEAD /messenger-attachments/<key> напрямую в S3 -> 403 (403)
|
||||
OK анонимный GET /messenger-attachments?list-type=2 напрямую в S3 -> 403 (403)
|
||||
OK анонимный GET /messenger-attachments/ напрямую в S3 -> 403 (403)
|
||||
OK анонимный PUT /messenger-attachments/<key> напрямую в S3 -> 403 (403)
|
||||
OK анонимный DELETE /messenger-attachments/<key> напрямую в S3 -> 403 (403)
|
||||
OK объект не изменён анонимными PUT/DELETE
|
||||
OK S3 не опубликован на хосте (docker port пуст) ''
|
||||
OK LB не проксирует путь S3 (404)
|
||||
OK SeaweedFS master UI/API :9333 недоступен из сети стенда
|
||||
OK SeaweedFS filer UI/API :8888 недоступен из сети стенда
|
||||
OK SeaweedFS volume :8080 недоступен из сети стенда
|
||||
OK SeaweedFS master gRPC :19333 недоступен из сети стенда
|
||||
OK SeaweedFS filer gRPC :18888 недоступен из сети стенда
|
||||
OK SeaweedFS admin :23646 недоступен из сети стенда
|
||||
-- s3 остановлен
|
||||
OK /health/ready при отказе S3 -> 200 (S3 не критичен для чата) (200)
|
||||
OK сообщение со ссылкой на готовое вложение -> 201 (201)
|
||||
OK metadata из PostgreSQL -> 200 (200)
|
||||
OK upload -> 503 unavailable (503 unavailable, 6.13 s)
|
||||
OK upload отказал быстрее timeout LB (< 8 s) (6.13 s)
|
||||
OK download -> 503 unavailable (503 unavailable, 6.01 s)
|
||||
OK download отказал быстрее timeout LB (< 8 s) (6.01 s)
|
||||
OK пользовательская ошибка при отказе S3 остаётся 415 (415)
|
||||
upload_intents после неудачных upload: 2
|
||||
-- s3 запущен
|
||||
OK s3 снова healthy
|
||||
OK после восстановления S3 upload -> 201 (201)
|
||||
OK старый файл скачивается побайтно (200)
|
||||
PASS: bucket приватен вне API, отказ S3 даёт 503 только для вложений
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,11 @@
|
||||
OK db_error_after_s3_put: 503 (503)
|
||||
OK db_error_after_s3_put: компенсация убрала intent
|
||||
OK crash_after_s3_put: клиент не получил 201 (RemoteDisconnected)
|
||||
OK crash_after_s3_put: остались intent и объект
|
||||
OK janitor до grace ничего не удаляет
|
||||
-- ждём grace 21 s
|
||||
OK janitor после grace удалил intent от crash (cleaned=1)
|
||||
OK janitor удалил объект от crash
|
||||
OK свежий intent не тронут
|
||||
OK готовое вложение цело
|
||||
PASS: компенсация и janitor не оставляют вечных объектов
|
||||
@@ -0,0 +1,3 @@
|
||||
OK upload через bb5edb449d7c -> bb5edb449d7c остановлена -> download через ['f00b790a44e9']: SHA-256 совпал
|
||||
OK upload через f00b790a44e9 -> f00b790a44e9 остановлена -> download через ['bb5edb449d7c']: SHA-256 совпал
|
||||
PASS: файл доступен после остановки любой обслужившей реплики
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"concurrency": 12,
|
||||
"file_bytes": 9962537,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"per_instance": {
|
||||
"b4841ddecd64": 6,
|
||||
"dd76ecc96935": 6
|
||||
},
|
||||
"wall_seconds": 5.18,
|
||||
"latency_p50_s": 4.98,
|
||||
"latency_max_s": 5.17,
|
||||
"sha256_ok": true,
|
||||
"memory_mib_before": {
|
||||
"dd76ecc96935": {
|
||||
"VmHWM": 169.2,
|
||||
"VmRSS": 105.2,
|
||||
"peak": 148.3,
|
||||
"current": 89.0,
|
||||
"anon": 87.5
|
||||
},
|
||||
"b4841ddecd64": {
|
||||
"VmHWM": 169.4,
|
||||
"VmRSS": 105.6,
|
||||
"peak": 148.7,
|
||||
"current": 85.2,
|
||||
"anon": 83.2
|
||||
}
|
||||
},
|
||||
"memory_mib_peak_during": {
|
||||
"dd76ecc96935": {
|
||||
"VmRSS_peak": 118.1,
|
||||
"anon_peak": 108.3,
|
||||
"samples": 57
|
||||
},
|
||||
"b4841ddecd64": {
|
||||
"VmRSS_peak": 119.3,
|
||||
"anon_peak": 109.5,
|
||||
"samples": 57
|
||||
}
|
||||
},
|
||||
"memory_mib_after": {
|
||||
"dd76ecc96935": {
|
||||
"VmHWM": 169.2,
|
||||
"VmRSS": 118.1,
|
||||
"peak": 167.7,
|
||||
"current": 98.5,
|
||||
"anon": 95.9
|
||||
},
|
||||
"b4841ddecd64": {
|
||||
"VmHWM": 169.4,
|
||||
"VmRSS": 119.3,
|
||||
"peak": 174.5,
|
||||
"current": 99.9,
|
||||
"anon": 97.2
|
||||
}
|
||||
},
|
||||
"rss_peak_increase_mib": {
|
||||
"dd76ecc96935": 12.9,
|
||||
"b4841ddecd64": 13.7
|
||||
},
|
||||
"anon_peak_increase_mib": {
|
||||
"dd76ecc96935": 20.8,
|
||||
"b4841ddecd64": 26.3
|
||||
},
|
||||
"s3_server_mib": {
|
||||
"VmRSS_before": 203.5,
|
||||
"VmRSS_peak": 546.1,
|
||||
"anon_peak": 495.1,
|
||||
"samples": 47
|
||||
},
|
||||
"naive_in_memory_estimate_mib": 114.0
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"concurrency": 20,
|
||||
"file_bytes": 9962537,
|
||||
"statuses": {
|
||||
"201": 20
|
||||
},
|
||||
"per_instance": {
|
||||
"dd76ecc96935": 10,
|
||||
"b4841ddecd64": 10
|
||||
},
|
||||
"wall_seconds": 9.89,
|
||||
"latency_p50_s": 8.58,
|
||||
"latency_max_s": 9.88,
|
||||
"sha256_ok": true,
|
||||
"memory_mib_before": {
|
||||
"dd76ecc96935": {
|
||||
"VmHWM": 169.4,
|
||||
"VmRSS": 105.6,
|
||||
"peak": 148.5,
|
||||
"current": 85.6,
|
||||
"anon": 83.1
|
||||
},
|
||||
"b4841ddecd64": {
|
||||
"VmHWM": 169.3,
|
||||
"VmRSS": 105.5,
|
||||
"peak": 148.8,
|
||||
"current": 85.8,
|
||||
"anon": 83.2
|
||||
}
|
||||
},
|
||||
"memory_mib_peak_during": {
|
||||
"dd76ecc96935": {
|
||||
"VmRSS_peak": 122.1,
|
||||
"anon_peak": 111.0,
|
||||
"samples": 98
|
||||
},
|
||||
"b4841ddecd64": {
|
||||
"VmRSS_peak": 123.2,
|
||||
"anon_peak": 113.8,
|
||||
"samples": 97
|
||||
}
|
||||
},
|
||||
"memory_mib_after": {
|
||||
"dd76ecc96935": {
|
||||
"VmHWM": 169.4,
|
||||
"VmRSS": 116.7,
|
||||
"peak": 195.2,
|
||||
"current": 97.8,
|
||||
"anon": 94.5
|
||||
},
|
||||
"b4841ddecd64": {
|
||||
"VmHWM": 169.3,
|
||||
"VmRSS": 123.2,
|
||||
"peak": 196.0,
|
||||
"current": 104.3,
|
||||
"anon": 101.4
|
||||
}
|
||||
},
|
||||
"rss_peak_increase_mib": {
|
||||
"dd76ecc96935": 16.5,
|
||||
"b4841ddecd64": 17.7
|
||||
},
|
||||
"anon_peak_increase_mib": {
|
||||
"dd76ecc96935": 27.9,
|
||||
"b4841ddecd64": 30.6
|
||||
},
|
||||
"s3_server_mib": {
|
||||
"VmRSS_before": 205.3,
|
||||
"VmRSS_peak": 778.8,
|
||||
"anon_peak": 698.4,
|
||||
"samples": 80
|
||||
},
|
||||
"naive_in_memory_estimate_mib": 190.0
|
||||
}
|
||||
@@ -81,6 +81,12 @@ backend api
|
||||
http-request disable-l7-retry if METH_POST !is_message_post
|
||||
http-request disable-l7-retry if METH_POST !has_idem_key
|
||||
http-request disable-l7-retry if METH_DELETE
|
||||
# Lab4: 503 вложения означает отказ общего S3 — на другой реплике тот же S3, повтор
|
||||
# только утроил бы ожидание клиента (3 × S3_OPERATION_TIMEOUT_SECONDS). conn-failure остаётся.
|
||||
http-request disable-l7-retry if { path_beg /api/v1/attachments/ }
|
||||
# Upload: после приёма тела API проверяет PNG/JPEG и делает S3 PUT с дедлайном
|
||||
# 6 s + size / 1 MiB/s (<= 16 s для 10 MiB) — 10 s «timeout server» мало.
|
||||
http-request set-timeout server 40s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ }
|
||||
|
||||
# До 8 реплик (scale/rolling update); пустые слоты остаются в MAINT, пока DNS не вернёт адрес.
|
||||
server-template api 8 api:8080
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{ "accessKey": "@ADMIN_ACCESS_KEY@", "secretKey": "@ADMIN_SECRET_KEY@" }
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{ "accessKey": "@API_ACCESS_KEY@", "secretKey": "@API_SECRET_KEY@" }
|
||||
],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"Write:messenger-attachments",
|
||||
"List:messenger-attachments"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# SeaweedFS all-in-one (master + volume + filer + S3 gateway) for the lab stand.
|
||||
#
|
||||
# - master/volume/filer and their web UIs listen on 127.0.0.1 INSIDE the
|
||||
# container only (-ip.bind): their HTTP API has no authentication and must
|
||||
# not be reachable from the Docker network. Only the S3 gateway (9000)
|
||||
# listens on all interfaces, and it requires SigV4 credentials.
|
||||
# - Identities and rights come from s3.json.template; the secret values are
|
||||
# substituted at start from Docker secrets into a file readable by root
|
||||
# only (never stored in Git or in the image).
|
||||
set -eu
|
||||
umask 077
|
||||
s() { cat "/run/secrets/$1"; }
|
||||
sed -e "s|@ADMIN_ACCESS_KEY@|$(s s3_admin_access_key_id)|" \
|
||||
-e "s|@ADMIN_SECRET_KEY@|$(s s3_admin_secret_access_key)|" \
|
||||
-e "s|@API_ACCESS_KEY@|$(s s3_access_key_id)|" \
|
||||
-e "s|@API_SECRET_KEY@|$(s s3_secret_access_key)|" \
|
||||
/opt/lab/s3.json.template > /tmp/s3.json
|
||||
|
||||
exec weed -logtostderr=true server \
|
||||
-dir=/data \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.volumeSizeLimitMB=256 -volume.max=0 \
|
||||
-s3 -s3.port=9000 -s3.ip.bind=0.0.0.0 -s3.config=/tmp/s3.json
|
||||
@@ -0,0 +1,68 @@
|
||||
-- Lab4: attachments. PostgreSQL keeps metadata, the bytes live in a private
|
||||
-- S3 bucket under a server-generated object_key. Expand-only migration: the
|
||||
-- lab3 API keeps working on this schema.
|
||||
|
||||
-- Text may be empty when the message has attachments; "not blank without
|
||||
-- attachments" is checked by the API (one message + its links are inserted
|
||||
-- in one transaction).
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank_check;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE RESTRICT,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
-- Display name only; never used as a path or an S3 key.
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
-- Lab4 has only "ready": the row is inserted after the object is stored.
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ready',
|
||||
error_code VARCHAR(64),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT attachments_content_type_check
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size_check
|
||||
CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256_check
|
||||
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('ready'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
-- A message references 0-10 attachments of its own chat; "position" keeps
|
||||
-- the order of attachment_ids. One attachment may be referenced by several
|
||||
-- messages of the chat.
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
position SMALLINT NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
|
||||
ON message_attachments (attachment_id);
|
||||
|
||||
-- Upload intent: committed BEFORE the S3 PUT and deleted in the same
|
||||
-- transaction that inserts the attachment row. An intent older than the
|
||||
-- grace period therefore means "the object may exist in S3, but the request
|
||||
-- never committed" (crash, lost DB, client gone) -> the janitor deletes the
|
||||
-- object and then the intent.
|
||||
CREATE TABLE IF NOT EXISTS upload_intents (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS upload_intents_created_idx
|
||||
ON upload_intents (created_at);
|
||||
+6
-1
@@ -1,7 +1,12 @@
|
||||
fastapi==0.115.0
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
pydantic==2.9.2
|
||||
SQLAlchemy==2.0.36
|
||||
asyncpg==0.30.0
|
||||
argon2-cffi==23.1.0
|
||||
redis==5.0.8
|
||||
# Lab4: S3 client (async) and image header/JPEG decoding
|
||||
aiobotocore==2.15.2
|
||||
botocore==1.35.36
|
||||
aiohttp==3.10.10
|
||||
Pillow==11.0.0
|
||||
|
||||
@@ -34,3 +34,10 @@ if [ ! -f "$dir/valkey_password.txt" ]; then
|
||||
fi
|
||||
valkey_password="$(cat "$dir/valkey_password.txt")"
|
||||
write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0"
|
||||
|
||||
# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init
|
||||
# bootstrap; the API gets its own key pair restricted to one bucket.
|
||||
write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)"
|
||||
write_new s3_admin_secret_access_key.txt "$(random)$(random)"
|
||||
write_new s3_access_key_id.txt "api$(random | cut -c1-17)"
|
||||
write_new s3_secret_access_key.txt "$(random)$(random)"
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Общие функции операторских скриптов. Используют только docker CLI и метки Compose,
|
||||
# поэтому работают и на хосте, и в контейнере tester (docker.sock).
|
||||
PROJECT="${COMPOSE_PROJECT:-messenger-lab3}"
|
||||
PROJECT="${COMPOSE_PROJECT:-messenger-lab4}"
|
||||
|
||||
# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные.
|
||||
service_ids() {
|
||||
|
||||
+11
-10
@@ -1,19 +1,20 @@
|
||||
#!/bin/sh
|
||||
# Persistence hook: restarts the LB, every API replica and both stores
|
||||
# (PostgreSQL with its volume, Valkey cache). No "down -v", no re-seed.
|
||||
# Persistence hook: restarts the LB, every API replica and all stores
|
||||
# (PostgreSQL and S3 with their volumes, Valkey cache). No "down -v", no re-seed.
|
||||
# Uses only the docker CLI and Compose labels, so it also runs in "tester".
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db); valkey=$(service_ids valkey)
|
||||
[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] || { echo "stack is not running" >&2; exit 2; }
|
||||
echo "stopping lb + api, then db + valkey"
|
||||
api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db)
|
||||
valkey=$(service_ids valkey); s3=$(service_ids s3)
|
||||
[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] && [ -n "$s3" ] || { echo "stack is not running" >&2; exit 2; }
|
||||
echo "stopping lb + api, then db + valkey + s3"
|
||||
docker stop --time 15 $lb $api >/dev/null
|
||||
docker stop --time 30 $db $valkey >/dev/null
|
||||
echo "starting db + valkey"
|
||||
docker start $db $valkey >/dev/null
|
||||
wait_healthy 60 $db $valkey
|
||||
docker stop --time 30 $db $valkey $s3 >/dev/null
|
||||
echo "starting db + valkey + s3"
|
||||
docker start $db $valkey $s3 >/dev/null
|
||||
wait_healthy 80 $db $valkey $s3
|
||||
echo "starting api"
|
||||
docker start $api >/dev/null
|
||||
wait_healthy 60 $api
|
||||
docker start $lb >/dev/null
|
||||
echo "restart complete: db=$(short $db) valkey=$(short $valkey) api=$(for i in $api; do short $i; done | tr '\n' ' ')"
|
||||
echo "restart complete: db=$(short $db) valkey=$(short $valkey) s3=$(short $s3) api=$(for i in $api; do short $i; done | tr '\n' ' ')"
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Idempotent bootstrap of the private attachments bucket (service s3-init).
|
||||
|
||||
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
|
||||
before any API replica starts:
|
||||
|
||||
1. wait until the S3 gateway answers;
|
||||
2. create the bucket if it does not exist (a repeated run prints "keep");
|
||||
3. self-checks, the stand does not start if one fails:
|
||||
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
|
||||
- API credentials can PUT/GET/DELETE under the prefix;
|
||||
- API credentials can NOT create/delete buckets or touch other buckets.
|
||||
|
||||
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import botocore.session
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
file_name = os.getenv(f"{name}_FILE")
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} or {name}_FILE is required")
|
||||
return value
|
||||
|
||||
|
||||
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
|
||||
|
||||
|
||||
def client(access_key: str, secret_key: str):
|
||||
return botocore.session.get_session().create_client(
|
||||
"s3",
|
||||
endpoint_url=ENDPOINT,
|
||||
region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
retries={"max_attempts": 1},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def error_code(exc: ClientError) -> str:
|
||||
return exc.response.get("Error", {}).get("Code", "")
|
||||
|
||||
|
||||
failures = 0
|
||||
|
||||
|
||||
def check(label: str, ok: bool) -> None:
|
||||
global failures
|
||||
failures += not ok
|
||||
print(("PASS " if ok else "FAIL ") + label, flush=True)
|
||||
|
||||
|
||||
def denied(call) -> bool:
|
||||
try:
|
||||
call()
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
|
||||
return False
|
||||
|
||||
|
||||
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
|
||||
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
|
||||
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
|
||||
|
||||
for attempt in range(1, 61):
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
if attempt == 60:
|
||||
raise
|
||||
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
admin.head_bucket(Bucket=BUCKET)
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
except ClientError as exc:
|
||||
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
try:
|
||||
admin.create_bucket(Bucket=BUCKET)
|
||||
print(f"create bucket {BUCKET}", flush=True)
|
||||
except ClientError as create_exc:
|
||||
# Two bootstrap runs at the same time: the other one won.
|
||||
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
|
||||
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
|
||||
|
||||
check("api credentials: PUT object", not denied(
|
||||
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
|
||||
check("api credentials: GET object",
|
||||
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
|
||||
|
||||
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
|
||||
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
|
||||
check("anonymous PUT object -> 403",
|
||||
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
|
||||
|
||||
other = f"forbidden-{uuid.uuid4().hex[:12]}"
|
||||
check("api credentials: CreateBucket denied",
|
||||
denied(lambda: api.create_bucket(Bucket=other)))
|
||||
check("api credentials: DeleteBucket denied",
|
||||
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
|
||||
check("api credentials: DELETE object", not denied(
|
||||
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
|
||||
|
||||
def foreign_buckets_hidden() -> bool:
|
||||
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
|
||||
# servers may answer with only the allowed bucket. Both are fine.
|
||||
try:
|
||||
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
|
||||
return all(name == BUCKET for name in names)
|
||||
|
||||
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
|
||||
|
||||
if failures:
|
||||
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
|
||||
return 1
|
||||
|
||||
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Негативные и граничные сценарии вложений lab4 (только stdlib).
|
||||
|
||||
Проверяет, что пользовательская ошибка никогда не становится 500:
|
||||
- размер: ровно 10 MiB -> 201; 10 MiB + 1 по Content-Length и chunked -> 413;
|
||||
- пиксели: ровно 40 млн -> 201; 50 млн (маленький файл, «бомба») -> 413;
|
||||
- повреждённые PNG/JPEG (обрезка, CRC, мусор в IDAT), чужие форматы -> 415;
|
||||
- подмена MIME: PNG как image/jpeg, JPEG как image/png, text/plain, multipart -> 415;
|
||||
- опасные filename (/, \\, NUL, управляющие, bidi, пустой, 256 символов) -> 400;
|
||||
- чужой пользователь: upload в чужой чат, metadata, content -> 404;
|
||||
- message: вложение другого чата / неизвестное -> 404, дубли / 11 ID -> 400,
|
||||
text="" с вложением -> 201, повтор Idempotency-Key -> то же сообщение;
|
||||
- оборванная загрузка не ломает реплику; variant без варианта -> 404.
|
||||
|
||||
Запуск: docker compose --profile tools run --rm tester python3 student-tests/attachment_edge_cases.py
|
||||
или с хоста: python student-tests/attachment_edge_cases.py --base-url http://localhost:8080
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import urllib.parse
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAMPLE_PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
SAMPLE_JPG = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes()
|
||||
MIB10 = 10 * 1024 * 1024
|
||||
|
||||
|
||||
# --- синтетические изображения ---------------------------------------------
|
||||
|
||||
def png_chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def png(width: int, height: int, bit_depth: int = 8, color_type: int = 0, idat: bytes | None = None,
|
||||
extra: bytes = b'') -> bytes:
|
||||
if idat is None:
|
||||
row_bytes = (width * bit_depth * {0: 1, 2: 3}[color_type] + 7) // 8
|
||||
raw = (b'\x00' + b'\x00' * row_bytes) * height
|
||||
idat = zlib.compress(raw, 9)
|
||||
ihdr = struct.pack('>IIBBBBB', width, height, bit_depth, color_type, 0, 0, 0)
|
||||
return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', ihdr) + extra
|
||||
+ png_chunk(b'IDAT', idat) + png_chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def png_exactly(size: int) -> bytes:
|
||||
"""Валидный PNG ровно size байт: добивается приватным ancillary-чанком."""
|
||||
base = png(8, 8)
|
||||
pad = size - len(base) - 12
|
||||
return png(8, 8, extra=png_chunk(b'zpAD', b'\x00' * pad))
|
||||
|
||||
|
||||
def pixels_png(width: int, height: int) -> bytes:
|
||||
"""1-битный grayscale: 40 млн пикселей занимают ~5 MB raw, ~10 KB zlib."""
|
||||
row = b'\x00' + b'\x00' * ((width + 7) // 8)
|
||||
compressor = zlib.compressobj(9)
|
||||
idat = b''.join(compressor.compress(row) for _ in range(height)) + compressor.flush()
|
||||
return png(width, height, bit_depth=1, idat=idat)
|
||||
|
||||
|
||||
def corrupt_crc(data: bytes) -> bytes:
|
||||
i = data.index(b'IDAT') + 6
|
||||
return data[:i] + bytes([data[i] ^ 0xFF]) + data[i + 1:]
|
||||
|
||||
|
||||
def corrupt_idat(data: bytes) -> bytes:
|
||||
"""Мусор в zlib-потоке с ПРАВИЛЬНЫМ CRC: ловится только распаковкой."""
|
||||
return png(320, 200, color_type=2, idat=b'\x78\x9c' + os.urandom(2000))
|
||||
|
||||
|
||||
# --- HTTP ------------------------------------------------------------------
|
||||
|
||||
class Api:
|
||||
def __init__(self, base_url: str):
|
||||
parts = urllib.parse.urlsplit(base_url)
|
||||
self.host = parts.hostname
|
||||
self.port = parts.port or 80
|
||||
|
||||
def conn(self) -> http.client.HTTPConnection:
|
||||
return http.client.HTTPConnection(self.host, self.port, timeout=60)
|
||||
|
||||
def call(self, method, path, body=None, headers=None, raw=None, chunks=None):
|
||||
h = dict(headers or {})
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
h['Content-Type'] = 'application/json'
|
||||
c = self.conn()
|
||||
try:
|
||||
if chunks is not None:
|
||||
c.request(method, '/api/v1' + path, body=chunks, headers=h, encode_chunked=True)
|
||||
else:
|
||||
c.request(method, '/api/v1' + path, body=raw, headers=h)
|
||||
r = c.getresponse()
|
||||
data = r.read()
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
# Сервер ответил 413 и закрыл соединение, не дочитав тело.
|
||||
r = c.getresponse()
|
||||
data = r.read()
|
||||
finally:
|
||||
c.close()
|
||||
ctype = r.getheader('Content-Type', '')
|
||||
parsed = json.loads(data) if data and ctype.startswith('application/json') else None
|
||||
return r.status, parsed, data, r
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080'))
|
||||
args = p.parse_args()
|
||||
api = Api(args.base_url)
|
||||
failed = 0
|
||||
|
||||
def code(resp):
|
||||
return resp.get('error', {}).get('code') if isinstance(resp, dict) else None
|
||||
|
||||
def check(label, status, resp, want_status, want_code=None, extra_ok=True):
|
||||
nonlocal failed
|
||||
ok = status == want_status and (want_code is None or code(resp) == want_code) and extra_ok
|
||||
failed += not ok
|
||||
print(f"{'OK ' if ok else 'FAIL'} {label}: {status} {code(resp) or ''} "
|
||||
f"(ожидается {want_status} {want_code or ''})", flush=True)
|
||||
|
||||
def new_user():
|
||||
name = 'att' + uuid.uuid4().hex[:12]
|
||||
password = 'attachment-edge-password'
|
||||
s, user, _, _ = api.call('POST', '/users', {'username': name, 'display_name': 'Вложения', 'password': password})
|
||||
assert s == 201, f'register: {s}'
|
||||
basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode()
|
||||
s, session, _, _ = api.call('POST', '/auth/sessions', headers={'Authorization': basic})
|
||||
assert s == 201, f'login: {s}'
|
||||
return user, {'Authorization': 'Bearer ' + session['token']}
|
||||
|
||||
def new_chat(auth, members, title='attachments'):
|
||||
s, chat, _, _ = api.call('POST', '/chats', {'title': title, 'member_ids': members}, headers=auth)
|
||||
assert s == 201, f'chat: {s}'
|
||||
return chat
|
||||
|
||||
def upload(auth, chat_id, data, content_type='image/png', filename='photo.png', **kw):
|
||||
query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='')
|
||||
h = dict(auth)
|
||||
if content_type is not None:
|
||||
h['Content-Type'] = content_type
|
||||
return api.call('POST', f'/chats/{chat_id}/attachments{query}', headers=h, raw=data, **kw)
|
||||
|
||||
def message(auth, chat_id, text, ids=None, key=None):
|
||||
body = {'text': text}
|
||||
if ids is not None:
|
||||
body['attachment_ids'] = ids
|
||||
h = dict(auth, **{'Idempotency-Key': key or uuid.uuid4().hex})
|
||||
return api.call('POST', f'/chats/{chat_id}/messages', body, headers=h)
|
||||
|
||||
alice, a_auth = new_user()
|
||||
bob, b_auth = new_user()
|
||||
eve, e_auth = new_user()
|
||||
chat = new_chat(a_auth, [alice['id'], bob['id']])
|
||||
other_chat = new_chat(a_auth, [alice['id']], 'other')
|
||||
eve_chat = new_chat(e_auth, [eve['id']], 'eve')
|
||||
|
||||
# --- успешные границы ---------------------------------------------------
|
||||
s, att, _, r = upload(a_auth, chat['id'], SAMPLE_JPG, 'image/jpeg', 'фото 😀 v1.0.jpg')
|
||||
check('JPEG fixture -> 201 ready', s, att, 201, extra_ok=s == 201 and att['status'] == 'ready'
|
||||
and att['sha256'] == hashlib.sha256(SAMPLE_JPG).hexdigest() and att['size_bytes'] == len(SAMPLE_JPG)
|
||||
and att['filename'] == 'фото 😀 v1.0.jpg' and att['variants'] == [] and att['error_code'] is None
|
||||
and 'object_key' not in att)
|
||||
jpg_id = att['id']
|
||||
s, _, data, r = api.call('GET', f'/attachments/{jpg_id}/content', headers=b_auth)
|
||||
check('участник скачивает JPEG побайтно, без redirect', s, None, 200,
|
||||
extra_ok=data == SAMPLE_JPG and r.getheader('Content-Type') == 'image/jpeg')
|
||||
|
||||
exact = png_exactly(MIB10)
|
||||
s, att, _, _ = upload(a_auth, chat['id'], exact)
|
||||
check('PNG ровно 10 MiB -> 201', s, att, 201, extra_ok=s == 201 and att['size_bytes'] == MIB10)
|
||||
if s == 201:
|
||||
s, _, data, _ = api.call('GET', f"/attachments/{att['id']}/content", headers=a_auth)
|
||||
check('10 MiB скачан побайтно', s, None, 200, extra_ok=hashlib.sha256(data).digest() == hashlib.sha256(exact).digest())
|
||||
|
||||
s, att, _, _ = upload(a_auth, chat['id'], pixels_png(8000, 5000))
|
||||
check('PNG ровно 40 млн пикселей -> 201', s, att, 201)
|
||||
|
||||
# --- размер -------------------------------------------------------------
|
||||
s, r, _, _ = upload(a_auth, chat['id'], png_exactly(MIB10 + 1))
|
||||
check('10 MiB + 1 байт (Content-Length) -> 413', s, r, 413, 'payload_too_large')
|
||||
|
||||
def big_chunks():
|
||||
block = png_exactly(MIB10)[:1024 * 1024]
|
||||
for _ in range(11):
|
||||
yield block
|
||||
|
||||
s, r, _, _ = upload(a_auth, chat['id'], None, chunks=big_chunks())
|
||||
check('11 MiB chunked без Content-Length -> 413', s, r, 413, 'payload_too_large')
|
||||
|
||||
s, r, _, _ = upload(a_auth, chat['id'], pixels_png(10000, 5000))
|
||||
check('PNG-«бомба» 50 млн пикселей (~10 KB файл) -> 413', s, r, 413, 'payload_too_large')
|
||||
s, r, _, _ = upload(a_auth, chat['id'], png(0x7FFFFFFF, 0x7FFFFFFF, idat=b'x'))
|
||||
check('PNG 2^31-1 x 2^31-1 в IHDR -> 413', s, r, 413, 'payload_too_large')
|
||||
|
||||
s, r, _, _ = upload(a_auth, chat['id'], b'')
|
||||
check('пустое тело -> 400', s, r, 400, 'invalid_request')
|
||||
|
||||
# --- повреждённые и чужие форматы --------------------------------------
|
||||
for label, data, ctype in [
|
||||
('PNG обрезан пополам', SAMPLE_PNG[:len(SAMPLE_PNG) // 2], 'image/png'),
|
||||
('PNG с неверным CRC', corrupt_crc(SAMPLE_PNG), 'image/png'),
|
||||
('PNG с мусором в zlib при верном CRC', corrupt_idat(SAMPLE_PNG), 'image/png'),
|
||||
('PNG без IEND', SAMPLE_PNG[:-12], 'image/png'),
|
||||
('JPEG обрезан', SAMPLE_JPG[:len(SAMPLE_JPG) // 2], 'image/jpeg'),
|
||||
('JPEG только сигнатура', b'\xff\xd8\xff\xe0' + b'\x00' * 100, 'image/jpeg'),
|
||||
('GIF как image/png', b'GIF89a' + b'\x00' * 100, 'image/png'),
|
||||
('случайные байты как image/png', os.urandom(4096), 'image/png'),
|
||||
('HTML как image/jpeg', b'<html><script>alert(1)</script></html>', 'image/jpeg'),
|
||||
]:
|
||||
s, r, _, _ = upload(a_auth, chat['id'], data, ctype)
|
||||
check(label + ' -> 415', s, r, 415, 'unsupported_media_type')
|
||||
|
||||
# --- подмена MIME -------------------------------------------------------
|
||||
for label, data, ctype in [
|
||||
('PNG под Content-Type image/jpeg', SAMPLE_PNG, 'image/jpeg'),
|
||||
('JPEG под Content-Type image/png', SAMPLE_JPG, 'image/png'),
|
||||
('PNG под text/plain', SAMPLE_PNG, 'text/plain'),
|
||||
('PNG под image/gif', SAMPLE_PNG, 'image/gif'),
|
||||
('multipart/form-data', b'--x\r\n\r\n' + SAMPLE_PNG + b'\r\n--x--', 'multipart/form-data; boundary=x'),
|
||||
('без Content-Type', SAMPLE_PNG, None),
|
||||
]:
|
||||
s, r, _, _ = upload(a_auth, chat['id'], data, ctype)
|
||||
check(label + ' -> 415', s, r, 415, 'unsupported_media_type')
|
||||
|
||||
# --- filename -----------------------------------------------------------
|
||||
for label, name in [
|
||||
('../../etc/passwd', '../../etc/passwd'),
|
||||
('C:\\boot.png', 'C:\\boot.png'),
|
||||
('NUL в имени', 'a\x00.png'),
|
||||
('перевод строки', 'a\nb.png'),
|
||||
('ESC-последовательность', '\x1b[31mred.png'),
|
||||
('bidi override (gnp.exe)', 'photo\u202egnp.exe'),
|
||||
('только пробелы', ' '),
|
||||
('..', '..'),
|
||||
('256 символов', 'a' * 252 + '.png'),
|
||||
('пустой filename', ''),
|
||||
('нет filename', None),
|
||||
]:
|
||||
s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG, filename=name)
|
||||
check(f'filename {label} -> 400', s, r, 400, 'invalid_request')
|
||||
s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG, filename='a' * 251 + '.png')
|
||||
check('filename ровно 255 символов -> 201', s, r, 201)
|
||||
|
||||
# --- доступ -------------------------------------------------------------
|
||||
s, r, _, _ = upload({}, chat['id'], SAMPLE_PNG)
|
||||
check('upload без токена -> 401', s, r, 401, 'unauthorized')
|
||||
s, r, _, _ = upload(e_auth, chat['id'], SAMPLE_PNG)
|
||||
check('посторонний upload в чужой чат -> 404', s, r, 404, 'not_found')
|
||||
s, r, _, _ = upload(a_auth, str(uuid.uuid4()), SAMPLE_PNG)
|
||||
check('upload в несуществующий чат -> 404', s, r, 404, 'not_found')
|
||||
for suffix in ('', '/content'):
|
||||
s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}{suffix}', headers=e_auth)
|
||||
check(f'посторонний GET /attachments/id{suffix} -> 404', s, r, 404, 'not_found')
|
||||
s, r, _, _ = api.call('GET', f'/attachments/{uuid.uuid4()}{suffix}', headers=a_auth)
|
||||
check(f'неизвестный attachment{suffix} -> 404', s, r, 404, 'not_found')
|
||||
s, r, _, _ = api.call('GET', '/attachments/not-a-uuid', headers=a_auth)
|
||||
check('attachment_id не UUID -> 400', s, r, 400, 'invalid_request')
|
||||
s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}/content?variant=thumbnail', headers=a_auth)
|
||||
check('ready без варианта thumbnail -> 404', s, r, 404, 'not_found')
|
||||
s, r, _, _ = api.call('GET', f'/attachments/{jpg_id}/content?variant=../../x', headers=a_auth)
|
||||
check('variant неизвестного имени -> 400', s, r, 400, 'invalid_request')
|
||||
|
||||
# --- привязка к сообщению -----------------------------------------------
|
||||
s, other_att, _, _ = upload(a_auth, other_chat['id'], SAMPLE_PNG)
|
||||
s, eve_att, _, _ = upload(e_auth, eve_chat['id'], SAMPLE_PNG)
|
||||
s, r, _, _ = message(a_auth, chat['id'], 'x', [other_att['id']])
|
||||
check('вложение другого чата (своего) -> 404', s, r, 404, 'not_found')
|
||||
s, r, _, _ = message(a_auth, chat['id'], 'x', [eve_att['id']])
|
||||
check('вложение чужого чата -> 404', s, r, 404, 'not_found')
|
||||
s, r, _, _ = message(a_auth, chat['id'], 'x', [jpg_id, str(uuid.uuid4())])
|
||||
check('одно из ID неизвестно -> 404, ничего не создано', s, r, 404, 'not_found')
|
||||
s, r, _, _ = message(a_auth, chat['id'], 'x', [jpg_id, jpg_id])
|
||||
check('дубли attachment_ids -> 400', s, r, 400, 'invalid_request')
|
||||
s, r, _, _ = message(a_auth, chat['id'], 'x', [str(uuid.uuid4()) for _ in range(11)])
|
||||
check('11 attachment_ids -> 400', s, r, 400, 'invalid_request')
|
||||
s, r, _, _ = message(a_auth, chat['id'], 'x', ['not-a-uuid'])
|
||||
check('attachment_id не UUID -> 400', s, r, 400, 'invalid_request')
|
||||
s, r, _, _ = message(a_auth, chat['id'], ' ', [])
|
||||
check('пробельный text без вложений -> 400', s, r, 400, 'invalid_request')
|
||||
|
||||
key = uuid.uuid4().hex
|
||||
s, m1, _, _ = message(b_auth, chat['id'], '', [jpg_id], key)
|
||||
check('Bob: text="" + вложение Alice из общего чата -> 201', s, m1, 201,
|
||||
extra_ok=s == 201 and m1['attachment_ids'] == [jpg_id] and m1['text'] == '')
|
||||
s, m2, _, _ = message(b_auth, chat['id'], '', [jpg_id], key)
|
||||
check('повтор Idempotency-Key -> то же сообщение', s, m2, 201, extra_ok=s == 201 and m2['id'] == m1['id'])
|
||||
s, r, _, _ = message(b_auth, chat['id'], 'другой текст', [], key)
|
||||
check('тот же ключ, другое тело -> 409', s, r, 409, 'conflict')
|
||||
s, page, _, _ = api.call('GET', f"/chats/{chat['id']}/messages?limit=100", headers=a_auth)
|
||||
linked = [m for m in page['items'] if m['attachment_ids']]
|
||||
check('история: ровно одно сообщение с вложением', s, page, 200,
|
||||
extra_ok=len(linked) == 1 and linked[0]['attachment_ids'] == [jpg_id])
|
||||
|
||||
# --- оборванная загрузка ------------------------------------------------
|
||||
sock = socket.create_connection((api.host, api.port), timeout=10)
|
||||
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\n"
|
||||
f"Host: {api.host}\r\nAuthorization: {a_auth['Authorization']}\r\n"
|
||||
f"Content-Type: image/png\r\nContent-Length: {len(SAMPLE_PNG)}\r\n\r\n")
|
||||
sock.sendall(head.encode() + SAMPLE_PNG[:len(SAMPLE_PNG) // 3])
|
||||
sock.close()
|
||||
s, r, _, _ = upload(a_auth, chat['id'], SAMPLE_PNG)
|
||||
check('после оборванной загрузки API работает', s, r, 201)
|
||||
|
||||
# --- формат ошибок ------------------------------------------------------
|
||||
s, r, _, resp = upload(a_auth, chat['id'], b'not an image')
|
||||
check('ошибка в общем формате с request_id', s, r, 415, extra_ok=bool(r)
|
||||
and r['error']['request_id'] == resp.getheader('X-Request-Id') and bool(resp.getheader('X-Instance-Id')))
|
||||
|
||||
print('PASS: граничные случаи вложений соответствуют контракту' if not failed
|
||||
else f'FAIL: {failed} проверок не прошло')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
@@ -0,0 +1,185 @@
|
||||
"""Приватность bucket вне API и недоступность S3 (stdlib + docker CLI).
|
||||
|
||||
1. Bucket приватен вне API: реальный object_key берётся из PostgreSQL
|
||||
(docker exec db psql), затем анонимные GET/HEAD/LIST/PUT напрямую в S3
|
||||
из сети backend -> 403; S3-порт не опубликован на хосте; LB не
|
||||
проксирует пути S3.
|
||||
2. Отказ S3 (docker stop s3): /health/ready 200, сообщения 201, metadata 200,
|
||||
upload/download -> быстрый 503 в общем формате (не 500, не зависание).
|
||||
После docker start s3: upload 201, старый файл скачивается побайтно.
|
||||
|
||||
Нужен docker.sock: запускайте только в tester на учебном стенде:
|
||||
docker compose --profile tools run --rm tester python3 student-tests/s3_failures.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/')
|
||||
S3 = os.environ.get('S3_ENDPOINT', 'http://s3:9000').rstrip('/')
|
||||
PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4')
|
||||
BUCKET = 'messenger-attachments'
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
failed = 0
|
||||
|
||||
|
||||
def check(label, ok, detail=''):
|
||||
global failed
|
||||
failed += not ok
|
||||
print(f"{'OK ' if ok else 'FAIL'} {label} {detail}".rstrip(), flush=True)
|
||||
|
||||
|
||||
def docker(*args, check_rc=True) -> str:
|
||||
return subprocess.run(['docker', *args], check=check_rc, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def container(service: str) -> str:
|
||||
ids = docker('ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', f'label=com.docker.compose.service={service}').split()
|
||||
assert ids, f'no container for {service}'
|
||||
return ids[0]
|
||||
|
||||
|
||||
def http(method, url, data=None, headers=None, timeout=30):
|
||||
req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, r.read(), time.monotonic() - started
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read(), time.monotonic() - started
|
||||
|
||||
|
||||
def api(method, path, body=None, auth=None, raw=None, ctype=None, extra=None):
|
||||
h = dict(extra or {})
|
||||
if auth:
|
||||
h['Authorization'] = auth
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
ctype = 'application/json'
|
||||
if ctype:
|
||||
h['Content-Type'] = ctype
|
||||
status, data, elapsed = http(method, BASE + '/api/v1' + path, raw, h)
|
||||
try:
|
||||
parsed = json.loads(data) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return status, parsed, data, elapsed
|
||||
|
||||
|
||||
def err(parsed):
|
||||
return parsed.get('error', {}).get('code') if isinstance(parsed, dict) else None
|
||||
|
||||
|
||||
def wait_healthy(cid, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy':
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
name = 's3f' + uuid.uuid4().hex[:12]
|
||||
password = 's3-failure-password'
|
||||
s, user, _, _ = api('POST', '/users', {'username': name, 'display_name': 'S3', 'password': password})
|
||||
basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode()
|
||||
s, session, _, _ = api('POST', '/auth/sessions', auth=basic)
|
||||
auth = 'Bearer ' + session['token']
|
||||
s, chat, _, _ = api('POST', '/chats', {'title': 's3', 'member_ids': [user['id']]}, auth=auth)
|
||||
s, att, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=a.png", auth=auth, raw=SAMPLE, ctype='image/png')
|
||||
assert s == 201, s
|
||||
|
||||
# --- 1. приватный bucket вне API ---------------------------------------
|
||||
db = container('db')
|
||||
key = docker('exec', db, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc',
|
||||
f"SELECT object_key FROM attachments WHERE id = '{att['id']}'")
|
||||
check('object_key есть в PostgreSQL и не совпадает с filename/ID',
|
||||
bool(key) and 'a.png' not in key and att['id'] not in key)
|
||||
check('API не раскрывает object_key', key not in json.dumps(att))
|
||||
for method, path, data in [
|
||||
('GET', f'/{BUCKET}/{key}', None),
|
||||
('HEAD', f'/{BUCKET}/{key}', None),
|
||||
('GET', f'/{BUCKET}?list-type=2', None),
|
||||
('GET', f'/{BUCKET}/', None),
|
||||
('PUT', f'/{BUCKET}/{key}', b'overwrite'),
|
||||
('DELETE', f'/{BUCKET}/{key}', None),
|
||||
]:
|
||||
s, _, _ = http(method, S3 + path, data)
|
||||
check(f'анонимный {method} {path.replace(key, "<key>")} напрямую в S3 -> 403', s == 403, f'({s})')
|
||||
s, _, data, _ = api('GET', f"/attachments/{att['id']}/content", auth=auth)
|
||||
check('объект не изменён анонимными PUT/DELETE', hashlib.sha256(data).digest() == hashlib.sha256(SAMPLE).digest())
|
||||
s3 = container('s3')
|
||||
ports = docker('port', s3, check_rc=False)
|
||||
check('S3 не опубликован на хосте (docker port пуст)', ports == '', repr(ports))
|
||||
s, _, _ = http('GET', BASE + f'/{BUCKET}/{key}')
|
||||
check('LB не проксирует путь S3', s == 404, f'({s})')
|
||||
host = urllib.parse.urlsplit(S3).hostname
|
||||
for port, role in [(9333, 'master UI/API'), (8888, 'filer UI/API'), (8080, 'volume'),
|
||||
(19333, 'master gRPC'), (18888, 'filer gRPC'), (23646, 'admin')]:
|
||||
try:
|
||||
socket.create_connection((host, port), timeout=2).close()
|
||||
reachable = True
|
||||
except OSError:
|
||||
reachable = False
|
||||
check(f'SeaweedFS {role} :{port} недоступен из сети стенда', not reachable)
|
||||
|
||||
# --- 2. отказ S3 -------------------------------------------------------
|
||||
docker('stop', '--time', '10', s3)
|
||||
print('-- s3 остановлен', flush=True)
|
||||
try:
|
||||
s, _, _ = http('GET', BASE + '/health/ready')
|
||||
check('/health/ready при отказе S3 -> 200 (S3 не критичен для чата)', s == 200, f'({s})')
|
||||
s, m, _, _ = api('POST', f"/chats/{chat['id']}/messages", {'text': 'без S3', 'attachment_ids': [att['id']]},
|
||||
auth=auth, extra={'Idempotency-Key': uuid.uuid4().hex})
|
||||
check('сообщение со ссылкой на готовое вложение -> 201', s == 201, f'({s})')
|
||||
s, _, _, _ = api('GET', f"/attachments/{att['id']}", auth=auth)
|
||||
check('metadata из PostgreSQL -> 200', s == 200, f'({s})')
|
||||
s, p, _, t = api('POST', f"/chats/{chat['id']}/attachments?filename=b.png", auth=auth, raw=SAMPLE, ctype='image/png')
|
||||
check('upload -> 503 unavailable', s == 503 and err(p) == 'unavailable', f'({s} {err(p)}, {t:.2f} s)')
|
||||
check('upload отказал быстрее timeout LB (< 8 s)', t < 8, f'({t:.2f} s)')
|
||||
s, p, _, t = api('GET', f"/attachments/{att['id']}/content", auth=auth)
|
||||
check('download -> 503 unavailable', s == 503 and err(p) == 'unavailable', f'({s} {err(p)}, {t:.2f} s)')
|
||||
check('download отказал быстрее timeout LB (< 8 s)', t < 8, f'({t:.2f} s)')
|
||||
s, p, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=c.png", auth=auth, raw=b'junk', ctype='image/png')
|
||||
check('пользовательская ошибка при отказе S3 остаётся 415', s == 415, f'({s})')
|
||||
intents = docker('exec', db, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc',
|
||||
'SELECT count(*) FROM upload_intents')
|
||||
print(f' upload_intents после неудачных upload: {intents}', flush=True)
|
||||
finally:
|
||||
docker('start', s3)
|
||||
print('-- s3 запущен', flush=True)
|
||||
|
||||
check('s3 снова healthy', wait_healthy(s3))
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
s, new, _, _ = api('POST', f"/chats/{chat['id']}/attachments?filename=d.png", auth=auth, raw=SAMPLE, ctype='image/png')
|
||||
if s == 201 or time.monotonic() > deadline:
|
||||
break
|
||||
time.sleep(1)
|
||||
check('после восстановления S3 upload -> 201', s == 201, f'({s})')
|
||||
s, _, data, _ = api('GET', f"/attachments/{att['id']}/content", auth=auth)
|
||||
check('старый файл скачивается побайтно', s == 200 and data == SAMPLE, f'({s})')
|
||||
|
||||
print('PASS: bucket приватен вне API, отказ S3 даёт 503 только для вложений' if not failed
|
||||
else f'FAIL: {failed} проверок не прошло')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Уровень 4: отказ между S3 PUT и COMMIT не оставляет вечных объектов.
|
||||
|
||||
1. db_error_after_s3_put — процесс API (FAULT_INJECTION, порт 8091 внутри
|
||||
контейнера реплики) теряет БД после PUT -> 503, фоновая компенсация
|
||||
сама удаляет объект и intent;
|
||||
2. crash_after_s3_put — процесс (порт 8090) умирает после PUT до COMMIT ->
|
||||
остаются intent и объект; janitor после grace удаляет оба;
|
||||
3. свежий intent (моложе grace, «загрузка в процессе») janitor не трогает;
|
||||
готовое вложение после janitor скачивается побайтно.
|
||||
|
||||
docker compose --profile tools run --rm tester python3 student-tests/upload_compensation.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/')
|
||||
PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4')
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
GRACE = 20
|
||||
|
||||
failed = 0
|
||||
|
||||
|
||||
def check(label, ok, detail=''):
|
||||
global failed
|
||||
failed += not ok
|
||||
print(f"{'OK ' if ok else 'FAIL'} {label} {detail}".rstrip(), flush=True)
|
||||
|
||||
|
||||
def docker(*args, stdin=None) -> str:
|
||||
r = subprocess.run(['docker', *args], capture_output=True, text=True, input=stdin)
|
||||
if r.returncode:
|
||||
raise RuntimeError(f'docker {args[0]} failed: {r.stderr.strip()[-300:]}')
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def container(service: str) -> str:
|
||||
return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', f'label=com.docker.compose.service={service}').split()[0]
|
||||
|
||||
|
||||
API = DB = ''
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return docker('exec', DB, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', sql)
|
||||
|
||||
|
||||
def intents() -> set[str]:
|
||||
return set(filter(None, psql('SELECT object_key FROM upload_intents').splitlines()))
|
||||
|
||||
|
||||
def s3_exists(*keys: str) -> list:
|
||||
code = """
|
||||
import asyncio, json, sys
|
||||
from app.storage import storage
|
||||
async def main():
|
||||
await storage.start()
|
||||
out = [await storage.exists(k) for k in json.load(sys.stdin)]
|
||||
await storage.close()
|
||||
print(json.dumps(out))
|
||||
asyncio.run(main())
|
||||
"""
|
||||
return json.loads(docker('exec', '-i', API, 'python', '-c', code, stdin=json.dumps(list(keys))))
|
||||
|
||||
|
||||
def run_janitor(grace: int) -> int:
|
||||
code = f"""
|
||||
import asyncio
|
||||
from app.attachments import cleanup_stale_uploads
|
||||
from app.storage import storage
|
||||
async def main():
|
||||
await storage.start()
|
||||
print(await cleanup_stale_uploads({grace}))
|
||||
await storage.close()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
return int(docker('exec', API, 'python', '-c', code).splitlines()[-1])
|
||||
|
||||
|
||||
def http(method, url, data=None, headers=None, timeout=30):
|
||||
req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read()
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base):
|
||||
self.base = base
|
||||
|
||||
def call(self, method, path, body=None, auth=None, raw=None, ctype=None):
|
||||
h = {}
|
||||
if auth:
|
||||
h['Authorization'] = auth
|
||||
if body is not None:
|
||||
raw, ctype = json.dumps(body).encode(), 'application/json'
|
||||
if ctype:
|
||||
h['Content-Type'] = ctype
|
||||
s, data = http(method, self.base + '/api/v1' + path, raw, h)
|
||||
try:
|
||||
return s, json.loads(data)
|
||||
except ValueError:
|
||||
return s, data
|
||||
|
||||
def upload(self, auth, chat_id):
|
||||
return self.call('POST', f'/chats/{chat_id}/attachments?filename=c.png',
|
||||
auth=auth, raw=SAMPLE, ctype='image/png')
|
||||
|
||||
|
||||
def start_faulty(mode: str, port: int) -> Client:
|
||||
docker('exec', '-d', '-e', f'FAULT_INJECTION={mode}', '-e', f'APP_PORT={port}',
|
||||
'-e', 'JANITOR_INTERVAL_SECONDS=3600', API, 'python', '-m', 'app.server')
|
||||
name = docker('inspect', '-f', '{{.Name}}', API).lstrip('/')
|
||||
base = f'http://{name}:{port}'
|
||||
for _ in range(60):
|
||||
try:
|
||||
if http('GET', base + '/health/ready', timeout=2)[0] == 200:
|
||||
return Client(base)
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f'faulty process {mode} did not start')
|
||||
|
||||
|
||||
def stop_faulty() -> None:
|
||||
"""Kill the extra app.server processes (not PID 1, the real replica)."""
|
||||
code = """
|
||||
import os, signal
|
||||
for p in filter(str.isdigit, os.listdir('/proc')):
|
||||
if int(p) in (1, os.getpid()):
|
||||
continue
|
||||
try:
|
||||
cmd = open(f'/proc/{p}/cmdline', 'rb').read()
|
||||
except OSError:
|
||||
continue
|
||||
if b'app.server' in cmd:
|
||||
os.kill(int(p), signal.SIGKILL)
|
||||
"""
|
||||
docker('exec', API, 'python', '-c', code)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
global API, DB
|
||||
API, DB = container('api'), container('db')
|
||||
lb = Client(BASE)
|
||||
|
||||
name = 'comp' + uuid.uuid4().hex[:12]
|
||||
password = 'compensation-password-1'
|
||||
lb.call('POST', '/users', {'username': name, 'display_name': 'Comp', 'password': password})
|
||||
basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode()
|
||||
auth = 'Bearer ' + lb.call('POST', '/auth/sessions', auth=basic)[1]['token']
|
||||
me = lb.call('GET', '/users/me', auth=auth)[1]
|
||||
chat = lb.call('POST', '/chats', {'title': 'compensation', 'member_ids': [me['id']]}, auth=auth)[1]
|
||||
ready = lb.upload(auth, chat['id'])[1]
|
||||
|
||||
try:
|
||||
# 1. DB lost after PUT -> 503 + background compensation
|
||||
before = intents()
|
||||
s, _ = start_faulty('db_error_after_s3_put', 8091).upload(auth, chat['id'])
|
||||
time.sleep(2)
|
||||
check('db_error_after_s3_put: 503', s == 503, f'({s})')
|
||||
check('db_error_after_s3_put: компенсация убрала intent', intents() - before == set())
|
||||
|
||||
# 2. crash after PUT -> intent + object stay
|
||||
before = intents()
|
||||
try:
|
||||
s, _ = start_faulty('crash_after_s3_put', 8090).upload(auth, chat['id'])
|
||||
result = f'HTTP {s}'
|
||||
except OSError as exc:
|
||||
result = type(exc).__name__
|
||||
crash_keys = sorted(intents() - before)
|
||||
check('crash_after_s3_put: клиент не получил 201', '201' not in result, f'({result})')
|
||||
check('crash_after_s3_put: остались intent и объект',
|
||||
len(crash_keys) == 1 and s3_exists(*crash_keys) == [True])
|
||||
finally:
|
||||
stop_faulty()
|
||||
|
||||
check('janitor до grace ничего не удаляет',
|
||||
run_janitor(GRACE) == 0 and set(crash_keys) <= intents())
|
||||
|
||||
print(f'-- ждём grace {GRACE + 1} s', flush=True)
|
||||
time.sleep(GRACE + 1)
|
||||
|
||||
# 3. fresh in-flight upload (younger than grace) must survive
|
||||
inflight = f'originals/{uuid.uuid4().hex}'
|
||||
psql(f"INSERT INTO upload_intents (object_key, attachment_id, chat_id) "
|
||||
f"VALUES ('{inflight}', '{uuid.uuid4()}', '{chat['id']}')")
|
||||
|
||||
cleaned = run_janitor(GRACE)
|
||||
left = intents()
|
||||
check('janitor после grace удалил intent от crash', cleaned >= 1 and not set(crash_keys) & left,
|
||||
f'(cleaned={cleaned})')
|
||||
check('janitor удалил объект от crash', s3_exists(*crash_keys) == [False])
|
||||
check('свежий intent не тронут', inflight in left)
|
||||
psql(f"DELETE FROM upload_intents WHERE object_key = '{inflight}'")
|
||||
|
||||
s, body = lb.call('GET', f"/attachments/{ready['id']}/content", auth=auth)
|
||||
check('готовое вложение цело', s == 200 and hashlib.sha256(body).hexdigest() == ready['sha256'])
|
||||
|
||||
print('PASS: компенсация и janitor не оставляют вечных объектов' if not failed
|
||||
else f'FAIL: {failed} проверок не прошли')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Upload -> остановка обслужившей реплики -> download через другую, для КАЖДОЙ реплики.
|
||||
|
||||
Публичный failover-тест останавливает ту реплику, что ему ответила (при
|
||||
round-robin — обычно одну и ту же). Здесь цель выбирается явно: файл
|
||||
загружается, пока X-Instance-Id не совпадёт с целевой репликой, затем она
|
||||
останавливается (docker stop, остаётся остановленной на время проверки),
|
||||
файл скачивается через LB и сравнивается по SHA-256; реплика запускается снова.
|
||||
|
||||
docker compose --profile tools run --rm tester python3 student-tests/upload_failover.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080').rstrip('/')
|
||||
PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4')
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
|
||||
def docker(*args) -> str:
|
||||
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def call(method, path, body=None, auth=None, raw=None, ctype=None):
|
||||
h = {}
|
||||
if auth:
|
||||
h['Authorization'] = auth
|
||||
if body is not None:
|
||||
raw, ctype = json.dumps(body).encode(), 'application/json'
|
||||
if ctype:
|
||||
h['Content-Type'] = ctype
|
||||
req = urllib.request.Request(BASE + '/api/v1' + path, data=raw, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.read(), r.headers.get('X-Instance-Id')
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read(), e.headers.get('X-Instance-Id')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
failed = 0
|
||||
replicas = docker('ps', '-q', '--no-trunc', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', 'label=com.docker.compose.service=api').split()
|
||||
assert len(replicas) >= 2, 'need >= 2 running api replicas'
|
||||
|
||||
name = 'ufo' + uuid.uuid4().hex[:12]
|
||||
password = 'upload-failover-password'
|
||||
call('POST', '/users', {'username': name, 'display_name': 'UFO', 'password': password})
|
||||
basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode()
|
||||
auth = 'Bearer ' + json.loads(call('POST', '/auth/sessions', auth=basic)[1])['token']
|
||||
me = json.loads(call('GET', '/users/me', auth=auth)[1])
|
||||
chat = json.loads(call('POST', '/chats', {'title': 'ufo', 'member_ids': [me['id']]}, auth=auth)[1])
|
||||
digest = hashlib.sha256(SAMPLE).hexdigest()
|
||||
|
||||
for target in replicas:
|
||||
alias = target[:12]
|
||||
for _ in range(20):
|
||||
s, body, instance = call('POST', f"/chats/{chat['id']}/attachments?filename=ufo.png",
|
||||
auth=auth, raw=SAMPLE, ctype='image/png')
|
||||
if s == 201 and target.startswith(instance):
|
||||
break
|
||||
else:
|
||||
print(f'FAIL could not hit replica {alias}')
|
||||
failed += 1
|
||||
continue
|
||||
att = json.loads(body)
|
||||
docker('stop', '--time', '15', target)
|
||||
time.sleep(1)
|
||||
try:
|
||||
served = set()
|
||||
ok = True
|
||||
for _ in range(4):
|
||||
s, data, instance = call('GET', f"/attachments/{att['id']}/content", auth=auth)
|
||||
served.add(instance)
|
||||
ok &= s == 200 and hashlib.sha256(data).hexdigest() == digest
|
||||
ok &= alias not in served
|
||||
failed += not ok
|
||||
print(f"{'OK ' if ok else 'FAIL'} upload через {alias} -> {alias} остановлена -> "
|
||||
f"download через {sorted(served)}: SHA-256 {'совпал' if ok else 'НЕ совпал'}", flush=True)
|
||||
finally:
|
||||
docker('start', target)
|
||||
for _ in range(60):
|
||||
if docker('inspect', '-f', '{{.State.Health.Status}}', target) == 'healthy':
|
||||
break
|
||||
time.sleep(1)
|
||||
time.sleep(3)
|
||||
|
||||
print('PASS: файл доступен после остановки любой обслужившей реплики' if not failed
|
||||
else f'FAIL: {failed}')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Эксперимент: пик памяти API при N одновременных загрузках (stdlib + docker CLI).
|
||||
|
||||
Гипотеза: память реплики ограничена конструкцией, а не числом/размером
|
||||
загрузок: тело пишется в SpooledTemporaryFile (в RAM <= UPLOAD_SPOOL_MEMORY_BYTES
|
||||
на загрузку), PNG проверяется потоково, одновременно <= MAX_CONCURRENT_UPLOADS
|
||||
на реплику. Ожидаем прирост пика RSS << N x размер файла.
|
||||
|
||||
Методика: перезапуск API-реплик (сброс high-water mark) -> базовые VmRSS/VmHWM ->
|
||||
N потоков одновременно (Barrier) загружают валидный PNG из шума ~9.5 MiB ->
|
||||
во время загрузок внутри каждой реплики каждые 100 мс читается VmRSS процесса
|
||||
API (/proc/1/status) и anon cgroup (без page cache временных файлов) -> пик
|
||||
минус базовое значение -> проверка SHA-256. VmHWM не годится: стартовый пик
|
||||
импорта библиотек (~170 MiB) выше пика загрузок.
|
||||
|
||||
docker compose --profile tools run --rm tester python3 student-tests/upload_memory.py --concurrency 12
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4')
|
||||
|
||||
|
||||
def docker(*args) -> str:
|
||||
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def service_containers(service: str) -> list[str]:
|
||||
return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', f'label=com.docker.compose.service={service}').split()
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
return service_containers('api')
|
||||
|
||||
|
||||
def memory(cid: str) -> dict:
|
||||
out = docker('exec', cid, 'sh', '-c',
|
||||
'grep -E "^Vm(RSS|HWM)" /proc/1/status; '
|
||||
'echo "peak $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0)"; '
|
||||
'echo "current $(cat /sys/fs/cgroup/memory.current)"; '
|
||||
'grep -E "^anon " /sys/fs/cgroup/memory.stat')
|
||||
values = {}
|
||||
for line in out.splitlines():
|
||||
name, value = line.replace(':', '').split()[:2]
|
||||
values[name] = int(value) * (1024 if name.startswith('Vm') else 1)
|
||||
return {k: round(v / 2 ** 20, 1) for k, v in values.items()}
|
||||
|
||||
|
||||
SAMPLER = ('while :; do r=$(grep VmRSS /proc/1/status | tr -s " " | cut -d" " -f2); '
|
||||
'a=$(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2); '
|
||||
'echo "$r $a"; sleep 0.1; done')
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self, cid: str):
|
||||
self.proc = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', SAMPLER],
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
self.rss = self.anon = 0
|
||||
self.samples = 0
|
||||
self.thread = threading.Thread(target=self._read, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def _read(self):
|
||||
for line in self.proc.stdout:
|
||||
try:
|
||||
rss_kib, anon = map(int, line.split())
|
||||
except ValueError:
|
||||
continue
|
||||
self.rss = max(self.rss, rss_kib * 1024)
|
||||
self.anon = max(self.anon, anon)
|
||||
self.samples += 1
|
||||
|
||||
def stop(self) -> dict:
|
||||
self.proc.terminate()
|
||||
self.proc.wait()
|
||||
return {'VmRSS_peak': round(self.rss / 2 ** 20, 1), 'anon_peak': round(self.anon / 2 ** 20, 1),
|
||||
'samples': self.samples}
|
||||
|
||||
|
||||
def noise_png(target_bytes: int) -> bytes:
|
||||
width = 1800
|
||||
height = target_bytes // (width * 3 + 1)
|
||||
raw = b''.join(b'\x00' + os.urandom(width * 3) for _ in range(height))
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||||
ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def call(base, method, path, body=None, auth=None, raw=None, ctype=None, timeout=120):
|
||||
h = {}
|
||||
if auth:
|
||||
h['Authorization'] = auth
|
||||
if body is not None:
|
||||
raw, ctype = json.dumps(body).encode(), 'application/json'
|
||||
if ctype:
|
||||
h['Content-Type'] = ctype
|
||||
req = urllib.request.Request(base + '/api/v1' + path, data=raw, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, r.read(), r.headers.get('X-Instance-Id')
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read(), e.headers.get('X-Instance-Id')
|
||||
|
||||
|
||||
def wait_ready(base, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base + '/health/ready', timeout=2) as r:
|
||||
if r.status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit('API is not ready')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://lb:8080'))
|
||||
p.add_argument('--concurrency', type=int, default=12)
|
||||
p.add_argument('--size-mib', type=float, default=9.5)
|
||||
p.add_argument('--no-restart', action='store_true')
|
||||
p.add_argument('--output')
|
||||
args = p.parse_args()
|
||||
base = args.base_url.rstrip('/')
|
||||
|
||||
data = noise_png(int(args.size_mib * 2 ** 20))
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
print(f'file: {len(data)} bytes ({len(data) / 2 ** 20:.2f} MiB), concurrency {args.concurrency}', flush=True)
|
||||
|
||||
containers = api_containers()
|
||||
if not args.no_restart:
|
||||
docker('restart', '--time', '15', *containers)
|
||||
time.sleep(3)
|
||||
wait_ready(base)
|
||||
time.sleep(3)
|
||||
|
||||
name = 'mem' + uuid.uuid4().hex[:12]
|
||||
password = 'upload-memory-password'
|
||||
call(base, 'POST', '/users', {'username': name, 'display_name': 'Mem', 'password': password})
|
||||
basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode()
|
||||
s, session, _ = call(base, 'POST', '/auth/sessions', auth=basic)
|
||||
auth = 'Bearer ' + json.loads(session)['token']
|
||||
user = json.loads(call(base, 'GET', '/users/me', auth=auth)[1])
|
||||
chat = json.loads(call(base, 'POST', '/chats', {'title': 'mem', 'member_ids': [user['id']]}, auth=auth)[1])
|
||||
|
||||
before = {c[:12]: memory(c) for c in containers}
|
||||
samplers = {c[:12]: Sampler(c) for c in containers}
|
||||
s3 = service_containers('s3')[0]
|
||||
s3_before = memory(s3)
|
||||
s3_sampler = Sampler(s3)
|
||||
time.sleep(1)
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
results = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
t = time.monotonic()
|
||||
s, body, instance = call(base, 'POST', f"/chats/{chat['id']}/attachments?filename=noise.png",
|
||||
auth=auth, raw=data, ctype='image/png')
|
||||
with lock:
|
||||
results.append((s, time.monotonic() - t, instance, body))
|
||||
|
||||
started = time.monotonic()
|
||||
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(0.5)
|
||||
peaks = {c: sm.stop() for c, sm in samplers.items()}
|
||||
s3_peak = s3_sampler.stop()
|
||||
after = {c[:12]: memory(c) for c in containers}
|
||||
|
||||
ok = [r for r in results if r[0] == 201]
|
||||
sha_ok = all(json.loads(r[3])['sha256'] == digest for r in ok)
|
||||
if ok:
|
||||
att = json.loads(ok[0][3])
|
||||
s, content, _ = call(base, 'GET', f"/attachments/{att['id']}/content", auth=auth)
|
||||
sha_ok = sha_ok and hashlib.sha256(content).hexdigest() == digest
|
||||
latencies = sorted(r[1] for r in ok)
|
||||
summary = {
|
||||
'concurrency': args.concurrency,
|
||||
'file_bytes': len(data),
|
||||
'statuses': dict(collections.Counter(r[0] for r in results)),
|
||||
'per_instance': dict(collections.Counter(r[2] for r in results)),
|
||||
'wall_seconds': round(wall, 2),
|
||||
'latency_p50_s': round(latencies[len(latencies) // 2], 2) if latencies else None,
|
||||
'latency_max_s': round(latencies[-1], 2) if latencies else None,
|
||||
'sha256_ok': sha_ok,
|
||||
'memory_mib_before': before,
|
||||
'memory_mib_peak_during': peaks,
|
||||
'memory_mib_after': after,
|
||||
'rss_peak_increase_mib': {
|
||||
c: round(peaks[c]['VmRSS_peak'] - before[c]['VmRSS'], 1) for c in peaks
|
||||
},
|
||||
'anon_peak_increase_mib': {
|
||||
c: round(peaks[c]['anon_peak'] - before[c]['anon'], 1) for c in peaks
|
||||
},
|
||||
's3_server_mib': {'VmRSS_before': s3_before['VmRSS'], **s3_peak},
|
||||
'naive_in_memory_estimate_mib': round(args.concurrency * len(data) / 2 ** 20, 1),
|
||||
}
|
||||
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
||||
print(text)
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(text + '\n')
|
||||
return 0 if ok and sha_ok and all(r[0] in (201, 429) for r in results) else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user