From 9de64e36b2f1100a018dbd69616249be11ef6555 Mon Sep 17 00:00:00 2001 From: Ilya <145450507+IIcon18@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:09:08 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=92=D1=8B=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=9B=D0=A05=20=D0=BD=D0=B0=205=20(=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=BA=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 10 + .github/pull_request_template.md | 28 +- DEMO.md | 52 ++++ Dockerfile | 23 ++ Makefile | 34 +++ REPORT.md | 149 +++++++++- app/__init__.py | 0 app/auth.py | 113 +++++++ app/broker.py | 135 +++++++++ app/cache.py | 125 ++++++++ app/config.py | 95 ++++++ app/errors.py | 31 ++ app/faults.py | 67 +++++ app/images.py | 79 +++++ app/imaging.py | 224 ++++++++++++++ app/jobs.py | 103 +++++++ app/main.py | 378 ++++++++++++++++++++++++ app/migrate.py | 63 ++++ app/models.py | 98 +++++++ app/objects.py | 184 ++++++++++++ app/ops.py | 253 ++++++++++++++++ app/relay.py | 120 ++++++++ app/s3_bootstrap.py | 76 +++++ app/storage.py | 424 +++++++++++++++++++++++++++ app/sweeper.py | 162 ++++++++++ app/worker.py | 377 ++++++++++++++++++++++++ compose.yaml | 360 +++++++++++++++++++++++ evidence/README.md | 15 + evidence/failover.txt | 24 ++ evidence/faults.json | 133 +++++++++ evidence/faults.txt | 53 ++++ evidence/limits.txt | 9 + evidence/load.json | 17 ++ evidence/persistence.txt | 33 +++ evidence/pipeline.txt | 12 + evidence/smoke.txt | 6 + evidence/stop-one.txt | 2 + evidence/sweeper.txt | 26 ++ infra/haproxy/haproxy.cfg | 52 ++++ migrations/0001_init.sql | 49 ++++ migrations/0002_idempotency.sql | 12 + migrations/0003_attachments.sql | 49 ++++ migrations/0004_jobs_outbox.sql | 94 ++++++ requirements.txt | 9 + scripts/init-secrets.sh | 60 ++++ scripts/restart.sh | 10 + scripts/restore.sh | 5 + scripts/start-worker.sh | 5 + scripts/stop-one.sh | 19 ++ scripts/stop-worker.sh | 6 + student-tests/faults_test.py | 368 +++++++++++++++++++++++ student-tests/fixtures/alpha.png | Bin 0 -> 170 bytes student-tests/fixtures/exif_rot6.jpg | Bin 0 -> 2304 bytes student-tests/fixtures/sample.jpg | Bin 0 -> 3973 bytes student-tests/fixtures/wide.png | Bin 0 -> 4149 bytes student-tests/limits_test.py | 73 +++++ student-tests/pipeline_test.py | 139 +++++++++ student-tests/stack.py | 119 ++++++++ 58 files changed, 5132 insertions(+), 30 deletions(-) create mode 100644 DEMO.md create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/auth.py create mode 100644 app/broker.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/images.py create mode 100644 app/imaging.py create mode 100644 app/jobs.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/models.py create mode 100644 app/objects.py create mode 100644 app/ops.py create mode 100644 app/relay.py create mode 100644 app/s3_bootstrap.py create mode 100644 app/storage.py create mode 100644 app/sweeper.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 evidence/failover.txt create mode 100644 evidence/faults.json create mode 100644 evidence/faults.txt create mode 100644 evidence/limits.txt create mode 100644 evidence/load.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/pipeline.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stop-one.txt create mode 100644 evidence/sweeper.txt create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_jobs_outbox.sql create mode 100644 requirements.txt create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/start-worker.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/stop-worker.sh create mode 100644 student-tests/faults_test.py create mode 100644 student-tests/fixtures/alpha.png create mode 100644 student-tests/fixtures/exif_rot6.jpg create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/fixtures/wide.png create mode 100644 student-tests/limits_test.py create mode 100644 student-tests/pipeline_test.py create mode 100644 student-tests/stack.py diff --git a/.env.example b/.env.example index 119384b5..5797f127 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,13 @@ WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +# --- implementation-specific (lab5), see REPORT.md --- +WORKER_REPLICAS=1 +LEASE_SECONDS=20 +RETRY_BASE_SECONDS=2 +RETRY_MAX_SECONDS=60 +WORKER_MEMORY_LIMIT_MB=768 +PIPELINE_VERSION=1 +ORPHAN_GRACE_SECONDS=900 +# 1 only on a test stand: enables fault points armed via `python -m app.ops fault arm` +FAULT_INJECTION=0 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..5932c901 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Кошелев, БВТ2304 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 2 — доказанная идемпотентность pipeline (versioned pipeline, автоматический fault injection, инварианты I1–I5) -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: из lab4 — весь API, сессии, идемпотентность сообщений, LB, S3. Новое: upload 202 (staging в S3 + attachment/job/outbox одной транзакцией), RabbitMQ quorum queue + DLQ, outbox relay с publisher confirms, отдельный worker (lease, heartbeat, fencing, дочерний процесс с RLIMIT_AS и таймаутом), варианты thumbnail/crop/WebP, retry с backoff+jitter, реестр failed и replay, reaper, sweeper staging, pipeline v1/v2. +Команды запуска из чистого clone: `./scripts/init-secrets.sh && cp .env.example .env && docker compose --env-file .env up -d --build --wait` (подробно — REPORT.md). +Результаты проверок и ссылка на REPORT.md: make check PASS, make test PASS, pipeline 7/7, faults 14/14, limits PASS, persistence PASS, failover PASS ×2 — evidence/ и REPORT.md. +Ограничения / известные ошибки: одиночные узлы PostgreSQL/RabbitMQ/S3 (без HA); восстановление после смерти worker ≈ lease 20 с + backoff; ICC не переносится; см. раздел «Ограничения» REPORT.md. +Источники и использование ИИ: документация RabbitMQ (reliability, confirms, quorum queues), Pillow, aio-pika; ИИ-ассистент Claude использовался при реализации, тестах и отчёте, решения проверены автором. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/DEMO.md b/DEMO.md new file mode 100644 index 00000000..d656d5dc --- /dev/null +++ b/DEMO.md @@ -0,0 +1,52 @@ +# Сценарий защиты — лабораторная 5 + +Стенд поднят (`make up`, в `.env` `FAULT_INJECTION=1`). Все команды — из корня репозитория. В консоль не выводятся токены и пароли. + +## 1. Публичный smoke + +```sh +make test # 202 -> ready, оригинал побайтно, настоящий JPEG 256x160 +make stats # jobs по статусам, outbox, глубина очереди, DLQ +``` + +## 2. Worker остановлен: приём продолжается, content = 409 + +```sh +make stop-worker +python3 student-tests/faults_test.py --only worker_down # автоматически: upload 202, сообщение с вложением, + # content 409, другие сообщения 201, рестарт RabbitMQ и API, + # старт worker -> ready (печатает ID вложения и глубину очереди) +``` + +Вручную, если преподаватель просит «руками»: `make stop-worker` → загрузить `tests/fixtures/sample.png` по `examples/requests.http` → `GET …/content` = 409 → `docker compose restart rabbitmq api` → `make stats` (сообщение в очереди сохранилось) → `make start-worker` → `GET /attachments/{id}` = ready. + +## 3. Отказы уровня 4 + +```sh +python3 student-tests/faults_test.py --only outbox_gap,relay_crash_after_confirm # разрыв COMMIT/publish +python3 student-tests/faults_test.py --only crash_after_s3,crash_after_commit # crash после side effect до ack +python3 student-tests/faults_test.py --only transient_backoff,exhausted_then_replay # backoff+jitter, failed, replay +python3 student-tests/faults_test.py --only long_running_not_hung # lease + heartbeat +make failed # реестр failed +docker compose exec ops python -m app.ops inspect # история событий, ключи S3 +``` + +Ручной replay: `docker compose exec ops python -m app.ops replay --actor <фамилия>`; на ready-вложении команда отвечает `refused` (код 3). + +## 4. Трек 2: идемпотентность + +```sh +python3 student-tests/faults_test.py --only duplicate,reorder,new_version,stale_version_race,poison_message +``` + +Что показать в выводе: `duplicate_noop`, `stale_attempt_noop`, порядок `ready v2` → `superseded v1`, неизменные ключи S3 (инвариант I3). + +## Ответы на вопросы защиты + +**Чем publisher confirm отличается от consumer ack?** Confirm (`basic.ack` от брокера издателю) — брокер принял сообщение и записал его в quorum queue (Raft-лог на диске); после него relay ставит `published_at`. Consumer ack (от worker брокеру) — потребитель закончил обработку и брокер может удалить сообщение; у нас ack отправляется только после COMMIT результата в PostgreSQL. Это две разные границы ответственности: confirm закрывает окно producer → broker, ack — broker → consumer. + +**Что сохранено в момент 202?** Полный оригинал в `staging/` (S3 проверил SHA-256), строка `attachments(queued)` с размером/SHA-256/ключами, `jobs(queued)` и строка `outbox` — всё одной транзакцией. Брокер в этот момент может ещё ничего не знать: публикацию гарантирует outbox. + +**Почему повторная доставка нормальна и где ключ идемпотентности?** At-least-once неизбежно даёт дубли (падение после confirm до отметки outbox, после COMMIT до ack). Ключ эффекта — `(attachment_id, pipeline_version, variant)`: это и ключи S3, и PK `attachment_variants`; логическая работа — `jobs(job_id)` с UNIQUE `(attachment_id, pipeline_version)`; попытка — `(job_id, attempt)`. Worker стартует попытку только из `queued` с `attempt+1`, всё остальное — no-op с ack. + +**Как отличить зависший worker от долго работающего?** Живой worker продлевает lease heartbeat'ом каждые 5 с (lease 20 с) — даже при обработке 30 с вторая попытка не стартует. Зависшее *преобразование* убивается по `JOB_TIMEOUT_SECONDS` в дочернем процессе. Умерший worker перестаёт продлевать lease — reaper переводит job в queued. Если «умерший» оживёт, его COMMIT отклонит fencing: job уже не его `(lease_owner, attempt)` → `fenced_out`. Поэтому двух неконтролируемых зафиксированных результатов быть не может. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b8408c2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.8-slim-bookworm + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations + +RUN useradd --system --no-create-home --uid 10001 appuser +USER appuser + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONUNBUFFERED=1 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:'+os.environ.get('APP_PORT','8080')+'/health/ready',timeout=2).status==200 else 1)" + +CMD ["sh", "-c", "exec uvicorn app.main:app --host ${APP_HOST} --port ${APP_PORT} --timeout-graceful-shutdown 8 --timeout-keep-alive 30"] diff --git a/Makefile b/Makefile index 5d86d171..b2b0f2b0 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,37 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --- student targets (lab5) --- +.PHONY: up down test-pipeline test-faults test-limits stop-worker start-worker stats failed sweep-dry-run + +up: + ./scripts/init-secrets.sh + docker compose --env-file .env up -d --build --wait + +down: + docker compose stop + +test-pipeline: + $(PYTHON) student-tests/pipeline_test.py --base-url "$(BASE_URL)" + +test-faults: + $(PYTHON) student-tests/faults_test.py --base-url "$(BASE_URL)" --output evidence/faults.json + +test-limits: + $(PYTHON) student-tests/limits_test.py --base-url "$(BASE_URL)" + +stop-worker: + ./scripts/stop-worker.sh + +start-worker: + ./scripts/start-worker.sh + +stats: + docker compose exec -T ops python -m app.ops stats + +failed: + docker compose exec -T ops python -m app.ops failed + +sweep-dry-run: + docker compose exec -T sweeper python -m app.sweeper --once --dry-run diff --git a/REPORT.md b/REPORT.md index e98f75e9..94ad07f6 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,154 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кошелев, БВТ2304 +- Целевая оценка: **5**, трек **2 — «Доказанная идемпотентность pipeline»** +- Commit/PR: ветка `solution/bvt2304-koshelev`, PR в `mtusi2026lab5` (main) +- Использованные источники, библиотеки и помощь ИИ: код перенесён из собственной lab4 (FastAPI, uvicorn, pydantic, asyncpg, redis-py, argon2-cffi, boto3, Pillow). Новое: **RabbitMQ 4.1** (quorum queues, publisher confirms, dead-lettering, `x-delivery-limit`), **aio-pika 9.5.8**, PostgreSQL `FOR UPDATE SKIP LOCKED` и `LISTEN/NOTIFY`, Pillow (`ImageOps.exif_transpose`, `Image.draft`, WebP). Документация: RabbitMQ Reliability / Confirms / Quorum Queues, transactional outbox (microservices.io), Pillow handbook. При реализации, тестах и отчёте использовался ИИ-ассистент Claude (Anthropic); решения проверены и объяснимы автором. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Engine + Compose v2 (проверено: Docker Desktop 28.0.1, Compose 2.33.1), Python 3.9+ для тестов (проверено 3.14, только stdlib), `make`. Стенд: Apple M3, 8 CPU, 16 GiB, Docker VM 8 CPU / 7.6 GiB. + +```sh +./scripts/init-secrets.sh # ./secrets/* (в .gitignore): пароли PG/Valkey/RabbitMQ, S3-ключи, broker_url, rabbitmq.conf +cp .env.example .env # для тестов отказов: FAULT_INJECTION=1 в .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +# postgres, valkey, s3, rabbitmq -> migrate, s3-init -> api x2, relay, worker, sweeper, ops -> lb +make check && make test # публичные проверки +python3 student-tests/pipeline_test.py # варианты, EXIF, alpha, лимиты, invalid_image +python3 student-tests/faults_test.py --output evidence/faults.json # 14 сценариев отказов (нужен FAULT_INJECTION=1) +python3 student-tests/limits_test.py # лимиты времени/памяти worker (пересоздаёт worker) +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh ; ./scripts/restore.sh +``` + +- Операторские действия: `./scripts/stop-worker.sh`, `./scripts/start-worker.sh`, CLI `docker compose exec ops python -m app.ops `. RabbitMQ management UI — только `http://127.0.0.1:15672` (логин `messenger`, пароль в `secrets/rabbitmq_password`). +- Bootstrap идемпотентен: `migrate` применяет только новые файлы под advisory lock; `s3-init` — «bucket already exists»; топологию брокера (exchange/queues/bindings) объявляют relay/worker/ops при каждом старте (`declare` идемпотентен). +- Безопасная остановка: `docker compose stop` / `docker compose down` (volumes сохраняются). **Намеренное удаление учебных данных:** `docker compose down -v` (удаляет `pgdata`, `s3data`, `rabbitmqdata`). + +| Роль | Service | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | `haproxy:3.2.4-alpine` | `/lb-health`; backend по `/health/ready` | — | `edge`, **единственный** публичный порт `127.0.0.1:8080` | — | +| API ×2 | `api` | `mtusi-messenger:lab5` (build) | `/health/ready` (PostgreSQL) | — (stateless) | `edge` + `data` | `database_url`, `cache_url`, S3-ключи API; **нет** `broker_url` | +| Outbox relay | `relay` | `mtusi-messenger:lab5` | heartbeat-файл < 30 с | — | `data` | `database_url`, `broker_url` | +| Worker | `worker` | `mtusi-messenger:lab5` | heartbeat reaper-цикла < 30 с | — | `data` | `database_url`, `broker_url`, S3-ключи | +| Брокер | `rabbitmq` | `rabbitmq:4.1.4-management-alpine` | `rabbitmq-diagnostics check_port_connectivity` | `rabbitmqdata` | `data`; UI только `127.0.0.1:15672` (сеть `mgmt`) | `rabbitmq.conf` как secret в `conf.d` | +| S3 | `s3` | `chrislusf/seaweedfs:4.51` | `/healthz` + data node в топологии | `s3data` | `data` | `seaweedfs_s3_config` | +| PostgreSQL | `postgres` | `postgres:17.6-alpine3.22` | `pg_isready` | `pgdata` | `data` | `postgres_password` | +| Кэш сессий | `valkey` | `valkey/valkey:8.1.4-alpine3.22` | `ping` | — | `data` | `valkey_conf` | +| Очистка | `sweeper` | `mtusi-messenger:lab5` | heartbeat | — | `data` | `database_url`, S3-ключи | +| Оператор | `ops` | `mtusi-messenger:lab5` | — (idle) | — | `data` | `database_url`, `broker_url`, S3-ключи | +| One-shot | `migrate`, `s3-init` | `mtusi-messenger:lab5` | exit 0 | — | `data` | `database_url` / `s3_admin_*` | + +Критичные зависимости `/health/ready`: только PostgreSQL (синхронный путь сообщений/сессий). Недоступность брокера, worker или relay **не** делает API неготовым: upload продолжает отвечать 202 (outbox копится), сообщения работают. Недоступность S3 даёт 503 только на upload/download. + +Соответствие `.env.example`: `BROKER_URL_FILE` → secret `broker_url` (relay/worker/ops); `WORKER_CONCURRENCY=2` → prefetch AMQP = число параллельных дочерних процессов; `JOB_MAX_ATTEMPTS=5`; `JOB_TIMEOUT_SECONDS=60` → жёсткий лимит одного преобразования (kill дочернего процесса); `STAGING_RETENTION_SECONDS=86400` → хранение staging failed-вложений для replay; `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` → API и worker. Дополнительно: `WORKER_REPLICAS=1`, `LEASE_SECONDS=20`, `HEARTBEAT_SECONDS=5`, `RETRY_BASE_SECONDS=2`, `RETRY_MAX_SECONDS=60`, `WORKER_MEMORY_LIMIT_MB=768` (RLIMIT_AS дочернего процесса), `STUCK_QUEUED_SECONDS=300`, `PIPELINE_VERSION=1`, `ORPHAN_GRACE_SECONDS=900`, `FAULT_INJECTION=0|1`. Остальные — как в lab4. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client ─► lb ─► api-1/api-2 ── (1) PUT staging/ ──────────────────────────────► S3 (private bucket) + │ (2) ONE tx: attachments(queued)+jobs+outbox ─┐ ▲ ▲ + └─ 202 queued only after COMMIT ▼ │ │ + PostgreSQL (источник истины) │ │ + relay: outbox ─SKIP LOCKED─► publish ─confirm─► RabbitMQ quorum queue │ │ + attachments.process │ │ + │ prefetch=2 │ │ + worker: claim(lease) → GET staging → child process ───┘ │ + (decode/resize, RLIMIT_AS, timeout) → PUT final ──┘ + → fenced COMMIT(variants+ready+job done) → ack → delete staging + reaper (в worker): истёкший lease → retry/failed; «потерянное» сообщение → re-enqueue + poison message → reject → attachments.dlx → attachments.dlq +``` + +**Перенесено из lab4 без изменений:** пользователи, сессии (PostgreSQL + кэш Valkey), чаты, сообщения, keyset-пагинация, идемпотентность сообщений, балансировка, `X-Instance-Id`, потоковый приём файла в spool, проверки filename/Content-Type/сигнатуры/пикселей до декодирования, приватный bucket, ограниченные S3-ключи. + +**Изменено в lab5 (объявленный переход 201 → 202):** + +1. **Приём (API, `app/main.py`).** После синхронных дешёвых проверок (сигнатура, размеры из заголовка, лимит пикселей, CRC чанков PNG) оригинал целиком кладётся в `staging/` с `x-amz-checksum-sha256` (S3 отвергает искажённые байты). Затем **одна транзакция** PostgreSQL: `attachments(status=queued, staging_key, object_key)` + `jobs(queued, attempt=0)` + `outbox(payload)` + `job_events(accepted)` + `pg_notify`. Только после COMMIT — `202 queued`. **Что сохранено в момент 202:** полный проверенный оригинал в S3 (staging), метаданные с SHA-256 и намерение обработки (job + outbox) в PostgreSQL. Полное декодирование и resize в API больше не выполняются: файл, прошедший синхронные проверки, но не декодируемый, становится `failed / invalid_image`. API не знает адрес брокера. +2. **Outbox relay (`app/relay.py`).** `SELECT … WHERE published_at IS NULL AND available_at <= now() FOR UPDATE SKIP LOCKED` → publish с ожиданием **publisher confirm** (`mandatory=True`, persistent) → `UPDATE published_at` → COMMIT. Пробуждение — `LISTEN outbox` (задержка ~10 мс), отложенные строки (backoff) — опрос раз в секунду. Падение после confirm, но до COMMIT — повторная публикация (дубль), что безопасно (см. трек). +3. **Брокер.** Quorum queue `attachments.process` (Raft-лог на диске, durable), сообщения persistent, consumer ack ручной, `prefetch = WORKER_CONCURRENCY`. Сообщение строго по `contracts/jobs.schema.json`: `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at` — без байтов, URL и credentials; worker берёт ключи S3/размер/SHA-256 из БД. Невалидное сообщение → `reject(requeue=false)` → `attachments.dlx` → `attachments.dlq`; `x-delivery-limit=10` защищает от бесконечного цикла redelivery. +4. **Worker (`app/worker.py`, `app/imaging.py`).** Захват: в транзакции `jobs FOR UPDATE`; стартует попытку только если `status=queued` и `message.attempt == jobs.attempt + 1`, иначе ack как no-op с событием (`duplicate_noop`, `stale_attempt_noop`, `lease_held_noop`, `superseded_noop` …). Захват ставит `processing`, `attempt+1`, `lease_owner`, `lease_until = now()+20s`; heartbeat продлевает lease каждые 5 с. Преобразование — в отдельном процессе (`forkserver`) с `RLIMIT_AS=768 MiB` и жёстким таймаутом 60 с (kill). Запись в S3 по **детерминированным ключам эффекта** `attachments//original`, `attachments//v/{thumbnail.jpg,crop.jpg,optimized.webp}`. Фиксация — **fenced COMMIT**: результат принимается, только если job всё ещё `processing` с тем же `(lease_owner, attempt)`; версия не ниже опубликованной; затем `attachment_variants` (PK `(attachment_id, pipeline_version, name)`), `attachments.status=ready`, `jobs.status=done`. **Ack — только после COMMIT.** Затем удаление staging (сбой здесь чинит sweeper). +5. **Варианты (pipeline v1).** thumbnail: JPEG q85, вписан в 256×256 без увеличения, округление вниз, ≥1 px (320×200 → 256×160); crop: JPEG q85, центрированный квадрат 128×128 (`ImageOps.fit`, маленькие увеличиваются); optimized: WebP q80 method 4, вписан в 1280×1280 без увеличения, alpha сохраняется. Для всех: EXIF-ориентация применяется к пикселям (`exif_transpose`), alpha у JPEG компонуется на белый, **EXIF/XMP/ICC/комментарии не записываются** (Pillow пишет их только при явной передаче; проверяется тестом — в вариантах нет APP1 `Exif`). Оригинал — побайтно исходный (со своим EXIF). JPEG декодируется с DCT-масштабированием (`draft`) — память пропорциональна целевому, а не исходному размеру; целевые размеры считаются от исходных, поэтому результат точный. +6. **Ошибки и повторы (`app/jobs.py`).** Постоянные (без повторов): `invalid_image`, `image_too_large` (MemoryError под RLIMIT_AS / превышение пикселей), `staging_missing`, `staging_corrupt` (байты не совпали с SHA-256 из БД), `unsupported_pipeline`. Временные: `s3_unavailable`, `processing_timeout`, `worker_process_died:*`, `lease_expired`, неожиданные исключения. Временная ошибка → `queued` с `next_attempt_at = now() + backoff` и outbox-строкой с тем же `available_at` (отложенная публикация без плагинов брокера). Backoff экспоненциальный с «equal jitter»: `d = min(60, 2·2^(n-1))`, задержка `d/2 + U(0, d/2)` → окна [1,2], [2,4], [4,8], [8,16] с. После 5 попыток — `failed / retry_exhausted`. +7. **Реестр failed и replay.** Реестр — `jobs WHERE status='failed'` (`python -m app.ops failed`), DLQ брокера — для невалидных сообщений. `python -m app.ops replay --actor <кто>` в одной транзакции: только для `failed` (для `done` — отказ с кодом 3 и событием `replay_refused`), `max_attempts += 5`, `attachments → queued`, outbox, событие `replay` с автором (audit). Тот же `job_id` и тот же ключ эффекта — повтор не создаёт второго набора. +8. **Lease / зависание (reaper в каждом worker, `SKIP LOCKED`).** `processing` с `lease_until < now()` → `lease_expired` → retry (или failed, если попытки исчерпаны). Долго работающий worker продлевает lease heartbeat'ом и не теряет его; **зависший** преобразователь убивается по `JOB_TIMEOUT_SECONDS` (процесс worker жив, но попытка завершается `processing_timeout`); **умерший** worker перестаёт продлевать lease. Дубли исключены fencing'ом: даже если старый владелец «оживёт» после истечения lease, его COMMIT отклоняется (`fenced_out`). Страховка: job `queued` дольше `STUCK_QUEUED_SECONDS` без неопубликованной/свежей outbox-строки переотправляется (`republished_stuck`) — на случай потери сообщения в брокере. +9. **Staging и retention (`app/sweeper.py`).** staging удаляется только после COMMIT ready; failed-staging хранится `STAGING_RETENTION_SECONDS` (сутки) для replay; staging без строки (сбой между PUT и COMMIT, 202 не отправлялся) и объекты `attachments/…`, на которые не ссылается ни одна зафиксированная строка (вытесненная версия, fenced попытка), удаляются после `ORPHAN_GRACE_SECONDS`, только если у вложения нет активной job. queued/processing не трогаются никогда. +10. **API-семантика.** `GET /attachments/{id}` показывает `queued/processing/ready/failed`, `variants` только у ready, `error_code` только у failed. `/content` для не-ready → 409, отсутствующий вариант ready → 404. В сообщение можно прикрепить queued/processing/ready; failed → 409; чужое/неизвестное → 404. + +Схема (`migrations/0004_jobs_outbox.sql`): `jobs` (UNIQUE `(attachment_id, pipeline_version)`, CHECK согласованности lease и статуса), `outbox`, `attachment_variants`, `job_events` (audit), `fault_injections`. ## Доказательства критериев +Все файлы — в `evidence/`. Дата прогона 2026-11-10, ветка `solution/bvt2304-koshelev`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `make test` | PASS: 202 → ready, оригинал и настоящий JPEG 256×160 | `smoke.txt` | +| **3:** минимум lab4 сохранён, upload 201 → 202 | smoke + persistence + failover | PASS (данные и сессия после рестарта всех stores; отказ реплики API) | `smoke.txt`, `persistence.txt`, `failover.txt`, `load.json` | +| **3:** 202 только после устойчивого приёма; остановка API не теряет работу | `faults_test.py` `worker_down` | после 202 API перезапущен — attachment `queued`, затем ready | `faults.txt` | +| **3:** отдельные broker и worker; durable; ack после эффекта; без синхронного resize | compose; `worker_down` | при остановленном worker upload 202, content 409, другие сообщения 201; ack после COMMIT (`worker.py`) | `faults.txt` | +| **3:** состояния queued/processing/ready/failed; original + thumbnail | `pipeline_test.py` | 7/7 PASS; оригинал побайтно, thumbnail 256×160 | `pipeline.txt` | +| **3:** ограниченный retry, failed с error_code; повтор не создаёт второго набора | `exhausted_then_replay`, `duplicate` | 5 попыток → `retry_exhausted`; дубли → `duplicate_noop`, 3 строки вариантов | `faults.txt` | +| **3:** демонстрация worker stop → upload → message → restart API → start worker → ready; рестарт broker с volume | `worker_down` | очередь до/после рестарта RabbitMQ: `queue_ready_messages=1`; после старта worker ready, thumbnail 256×160 | `faults.txt`, `faults.json` | +| **4:** crop 128×128, WebP; параметры качества/метаданных | `pipeline_test.py` | crop 128×128; WebP 320×200 / 1280×320 / 200×300 (EXIF); EXIF в вариантах отсутствует; alpha → белый | `pipeline.txt` | +| **4:** лимиты bytes / пикселей / времени / памяти worker | `pipeline_test.py`, `limits_test.py` | 413/415 до 202; 36 Мп ready за ~1 с; RLIMIT_AS 160 MiB → `image_too_large` за 1 попытку; таймаут 0.3 с → kill, 4 retry, `retry_exhausted` | `pipeline.txt`, `limits.txt` | +| **4:** transactional outbox; crash между COMMIT и publish | `outbox_gap`, `relay_crash_after_confirm` | relay остановлен: outbox-строка `published=false`, вложение queued → relay старт → ready; crash после confirm → повторная публикация → `duplicate_noop` | `faults.txt` | +| **4:** backoff + jitter; постоянные/временные; DLQ/реестр; безопасный replay | `transient_backoff`, `pipeline_test` corrupt, `exhausted_then_replay`, `poison_message` | задержки 1.66 с и 2.77 с в окнах [1,2]/[2,4]; invalid_image без повторов; реестр failed; replay → ready; replay ready → отказ; невалидное сообщение → DLQ 0→1 | `faults.txt`, `pipeline.txt` | +| **4:** остановка worker после записи объекта до ack; lease; prefetch; очистка staging | `crash_after_s3`, `crash_after_commit`, `crash_after_claim`, `long_running_not_hung`, `sweeper.txt` | crash после S3 → lease истёк → попытка 2 перезаписала те же 4 ключа; crash после COMMIT → no-op; 30 с работы при lease 20 с → 1 попытка; staging пуст после ready (I5); sweeper удалил 6 неиспользуемых объектов, повтор — 0 | `faults.txt`, `sweeper.txt` | +| **5, трек 2** | `faults_test.py` (14 сценариев + инварианты I1–I5) | 14/14 PASS, итог: `outbox_unpublished=0`, `processing=0` | `faults.txt`, `faults.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперименты и отказы — трек 2: доказанная идемпотентность pipeline -## Эксперименты и отказы +**Гипотеза.** При доставке *at-least-once* (дубли, переупорядочивание, падение процесса в любой точке, устаревшая работа старой версии) комбинация (а) состояния job в PostgreSQL как единственного арбитра, (б) fencing-токена `(job_id, attempt)` + lease, (в) детерминированных ключей эффекта `(attachment_id, pipeline_version, variant)` и (г) монотонного правила версий даёт **идемпотентный эффект**: ни одна подтверждённая (202) задача не теряется, у вложения ровно один набор вариантов опубликованной версии, и результат старой версии никогда не перезаписывает новый. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Versioned pipeline.** `PIPELINES = {1: …, 2: …}` (`app/imaging.py`): v2 отличается параметрами качества (JPEG q90, WebP q72 method 6) и пишет в свои ключи `v2/…`. `python -m app.ops reprocess --pipeline-version 2` создаёт отдельную job (UNIQUE `(attachment, version)`); вложение во время перерасчёта **остаётся ready** со старыми вариантами, переключение — атомарный COMMIT. Даунгрейд отклоняется. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Инварианты** (проверяются после каждого сценария функцией `check_invariants`, через API и CLI): + +| | Инвариант | Как проверяется | +| --- | --- | --- | +| I1 | Подтверждённая задача не теряется | каждый 202 доходит до ожидаемого финального статуса | +| I2 | Один результат | ровно 3 строки вариантов для опубликованной версии; метаданные API = строки БД; SHA-256 отдаваемых API байтов = опубликованному; оригинал не изменился | +| I3 | Нет дублей эффектов | множество ключей `attachments//…` в S3 = original + ключи зафиксированных строк (+ явно ожидаемые остатки вытесненной версии) | +| I4 | Версии монотонны | последовательность событий `ready` по версиям неубывающая, опубликована максимальная | +| I5 | Ничего не зависло | нет `processing`, outbox полностью опубликован, staging удалён | + +**Методика.** Стенд из `compose.yaml` с `FAULT_INJECTION=1`, 1 worker × concurrency 2, lease 20 с. Fault points (`app/faults.py`) взводятся строкой в `fault_injections` и срабатывают ровно один раз (атомарный `UPDATE … SKIP LOCKED`); `crash` = `os._exit(137)` (как SIGKILL, Docker перезапускает контейнер). Дубли и переупорядочивание создаются сырой публикацией в брокер в обход outbox (`ops republish --attempt N --pipeline-version V`). Всё автоматизировано: `python3 student-tests/faults_test.py` (повторяемо, каждый запуск создаёт новых пользователей/вложения; ~5 минут). + +**Результаты** (`evidence/faults.txt`, `evidence/faults.json`): + +| Воздействие (окно отказа) | Что ожидаем | Что наблюдали (события `job_events`) | Восстановление / потеря | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| worker остановлен; рестарт API и RabbitMQ с volume | сообщение и работа сохраняются | очередь 1 сообщение до и после рестарта брокера → `claimed, ready` | после старта worker ~1 с; потерь нет | +| после COMMIT до publish (relay остановлен) | outbox держит намерение | `published=false` → после старта relay `claimed, ready` | ~1 с после старта relay; потерь нет | +| после publish/confirm до отметки в outbox (crash relay) | дубль сообщения безопасен | `claimed, ready, duplicate_noop` | дубль = no-op, 1 попытка | +| после claim до side effect (crash worker) | lease истекает, повтор | `claimed, lease_held_noop, lease_expired, retry_scheduled, claimed, ready` | ~25 с (lease 20 с + backoff); варианты от попытки 2 | +| после записи S3 до COMMIT (crash worker) | повтор перезаписывает те же ключи | то же; ключей в S3 ровно 4 | ~24 с; второго набора нет | +| после COMMIT до ack (crash worker) | redelivery — no-op | `ready, duplicate_noop` | ~1.5 с; 1 попытка | +| 3 дубля завершённой job | no-op | 3× `duplicate_noop`, `updated_at` не изменился | — | +| reorder: попытка 1 повторно и «будущая» 4 до настоящей 2; затем поздняя копия 1 из outbox | stale отбрасываются | 3× `stale_attempt_noop`, ровно 2 `claimed` | — | +| устаревшая job после новой версии (v1-сообщение после ready v2) | v2 сохраняется | `duplicate_noop`, опубликована v2 | — | +| гонка версий: v1 медленная (delay 8 с перед COMMIT), v2 фиксируется первой | v1 не перезаписывает v2 | порядок `ready v2` → `superseded v1`; строк v1 нет; объекты v1 — неиспользуемые остатки, удалены sweeper'ом | — | +| 2 временные ошибки | backoff+jitter | задержки 1.66 с, 2.77 с; ready с попытки 3 | ~5 с | +| 5 временных ошибок + replay | failed → явный replay | `retry_exhausted` → `replay` (audit, автор) → ready; повторный replay ready отклонён | — | +| 30 с работы при lease 20 с | долгий ≠ зависший | 1 попытка, нет `lease_expired` | — | +| невалидное сообщение | не зациклить | DLQ 0 → 1 | — | + +Итог полного прогона: 14/14 PASS, финальное состояние `{"outbox_unpublished": 0, "processing": 0}`, в DLQ — только намеренно отправленные невалидные сообщения. + +**Почему это не exactly-once доставка.** Брокер доставляет сообщение *at least once*: дубли реально возникали (crash relay после confirm, crash worker после COMMIT до ack, ручные republish). Exactly-once достигается не для *доставки*, а для *наблюдаемого эффекта*: побочные эффекты идемпотентны (те же ключи, те же байты) или отвергаются (fencing, проверка версии), а решение «эффект зафиксирован» принимается одной транзакцией PostgreSQL. Publisher confirm лишь означает, что брокер принял ответственность за сообщение (граница producer → broker); consumer ack — что worker надёжно зафиксировал результат (граница broker → consumer); между ними неизбежно окно, где один из участников может упасть, и система обязана терпеть повтор. + +**Ограничения измерения и решения.** +- Детерминизм кодирования (повтор пишет те же байты по тому же ключу) верен для одной сборки Pillow/libjpeg/libwebp. При смене образа повтор незавершённой попытки может записать другие байты по тому же ключу — но строка БД всегда фиксируется вместе с SHA-256 *записанных этой попыткой* байтов, а fencing гарантирует, что фиксирует только владелец lease. Окно «старый владелец перезаписал объект уже после COMMIT нового» (процесс замёрз дольше lease, затем проснулся и сделал PUT) теоретически возможно при недетерминированном кодировании; полностью закрывается ключами, включающими номер попытки, ценой мусора до sweeper — сознательно не выбрано, т.к. контракт задаёт ключ эффекта `(attachment, version, variant)`. +- Восстановление после смерти worker занимает lease (20 с) + backoff; уменьшение lease ускоряет восстановление, но повышает риск ложного «зависания» при GC/IO-паузах (heartbeat каждые 5 с даёт запас ×4). +- Тест проверяет 1 экземпляр worker с concurrency 2; конкуренцию двух worker за один lease покрывает fencing-логика (`fenced_out`), но отдельного стохастического стресс-теста нет. +- Fault injection включается переменной окружения и отключён по умолчанию (`FAULT_INJECTION=0`); таблица `fault_injections` без включённого флага не читается. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается: остановка/рестарт API, worker, relay, RabbitMQ, PostgreSQL, S3 и Valkey с volumes (persistence-тест), падение процесса в любом окне из `attachment-states.md`, дубли и переупорядочивание сообщений, потеря сообщения после публикации (страховка `STUCK_QUEUED_SECONDS`). +- Не выдерживается: потеря volume PostgreSQL (источник истины) или S3; одиночные узлы RabbitMQ/PostgreSQL/S3 (quorum queue на одном узле даёт durability, но не HA). +- Учебные упрощения: reaper встроен в worker (при нуле worker задачи с истёкшим lease ждут первого запуска worker — что и нужно, обрабатывать всё равно некому); API не проверяет полную декодируемость (это работа worker → `invalid_image`); ICC-профиль не переносится и цвета не конвертируются в sRGB. +- Нагрузочное исследование очереди (трек 1) не проводилось; `load.json` — только регресс синхронного пути сообщений: `tests/load.py --base-url http://127.0.0.1:8080 --duration 15 --concurrency 4` → 262 успешных RPS, p50/p95/p99 = 3.3/6.9/60 мс, 4 ошибки из ~3950. Ошибки — клиентские `URLError` (сбой установки TCP-соединения): в логе HAProxy за тот же интервал все 3955 POST сообщений получили 201 при server-side `Ta` p50 2 мс, p99 13 мс, max 86 мс, ни одного 5xx. То есть секундные задержки и обрывы возникают на участке macOS → проброс порта Docker Desktop (клиент открывает новое соединение на каждый запрос), а не в приложении. В lab4 на свежем демоне было 471 RPS без ошибок; разница RPS объясняется теми же редкими многосекундными задержками соединения в closed-loop генераторе, поэтому как сравнение производительности эти числа не используются. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..298a65d6 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import asyncio +import base64 +import binascii +import hashlib +import secrets +from datetime import datetime, timezone +from typing import Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from starlette.concurrency import run_in_threadpool + +from app import config +from app.cache import REVOKED, SessionCache +from app.errors import BASIC_CHALLENGE, ApiError, unauthorized +from app.storage import Store, fmt_ts, user_row + +_hasher = PasswordHasher() +_hash_slots = asyncio.Semaphore(config.PASSWORD_HASH_CONCURRENCY) +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing") + + +async def hash_password(password: str) -> str: + async with _hash_slots: + return await run_in_threadpool(_hasher.hash, password) + + +async def verify_password(password_hash: str, password: str) -> bool: + def verify() -> bool: + try: + return _hasher.verify(password_hash, password) + except (VerificationError, InvalidHashError): + return False + + async with _hash_slots: + return await run_in_threadpool(verify) + + +def token_digest(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() + + +def parse_basic(header: Optional[str]) -> Tuple[str, str]: + bad = unauthorized("invalid credentials", BASIC_CHALLENGE) + if not header: + raise bad + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise bad + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + raise bad from None + username, sep, password = decoded.partition(":") + if not sep or not username.isascii() or not 12 <= len(password) <= 128: + raise bad + return username, password + + +def parse_bearer(header: Optional[str]) -> str: + if not header: + raise unauthorized("bearer token required") + scheme, _, token = header.partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or not token or not token.isascii() or len(token) > 256: + raise unauthorized("bearer token required") + return token + + +async def login(store: Store, header: Optional[str]) -> dict: + username, password = parse_basic(header) + row = await store.get_credentials(username) + if row is None: + await verify_password(_DUMMY_HASH, password) + raise unauthorized("invalid credentials", BASIC_CHALLENGE) + if not await verify_password(row["password_hash"], password): + raise unauthorized("invalid credentials", BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + expires_at = await store.create_session(token_digest(token), row["id"], config.SESSION_TTL_SECONDS) + return {"token": token, "token_type": "Bearer", "expires_at": fmt_ts(expires_at), "user": user_row(row)} + + +async def authenticate(store: Store, cache: SessionCache, header: Optional[str]) -> dict: + digest = token_digest(parse_bearer(header)) + cached = await cache.get(digest) + if cached == REVOKED: + raise unauthorized("session expired or revoked") + if isinstance(cached, dict): + expires_at = datetime.fromisoformat(cached["expires_at"].replace("Z", "+00:00")) + if expires_at > datetime.now(timezone.utc): + return cached["user"] + raise unauthorized("session expired or revoked") + + row = await store.find_session(digest) + if row is None: + raise unauthorized("session expired or revoked") + user = user_row(row) + remaining = int((row["expires_at"] - datetime.now(timezone.utc)).total_seconds()) + await cache.put(digest, {"user": user, "expires_at": fmt_ts(row["expires_at"])}, remaining) + return user + + +async def logout(store: Store, cache: SessionCache, header: Optional[str]) -> None: + await authenticate(store, cache, header) + digest = token_digest(parse_bearer(header)) + expires_at = await store.revoke_session(digest) + if expires_at is None: + raise unauthorized("session expired or revoked") + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + 1 + if not await cache.revoke(digest, remaining): + raise ApiError(503, "unavailable", "logout not confirmed by session cache, retry later") diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..a706b8ae --- /dev/null +++ b/app/broker.py @@ -0,0 +1,135 @@ +"""RabbitMQ topology, job message format (contracts/jobs.schema.json) and publishing with publisher confirms. + +Topology (declared idempotently by every producer/consumer): + exchange attachments (direct, durable) --attachment.process.v1--> queue attachments.process (quorum, durable) + queue attachments.process: x-dead-letter-exchange=attachments.dlx, x-delivery-limit (poison message guard) + exchange attachments.dlx (fanout, durable) --> queue attachments.dlq (quorum, durable) + +Quorum queues replicate/fsync messages before the publisher confirm, persistent delivery mode is used anyway. +The message carries only IDs and service fields, never image bytes, URLs or credentials. +""" +from __future__ import annotations + +import json +import re +import uuid +from datetime import datetime, timezone +from typing import Optional + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType, Message + +from app import config + +JOB_TYPE = "attachment.process.v1" +JOB_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", + "traceparent", "tracestate"} +REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} +_TRACEPARENT = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + + +class InvalidJob(ValueError): + pass + + +def job_payload(job_id: uuid.UUID, attachment_id: uuid.UUID, pipeline_version: int, attempt: int) -> dict: + return { + "schema_version": 1, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + } + + +def _int(value, minimum: int) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise InvalidJob("integer field out of range") + return value + + +def parse_job(body: bytes) -> dict: + """Strict validation against jobs.schema.json (additionalProperties=false).""" + try: + data = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidJob("not JSON") from None + if not isinstance(data, dict): + raise InvalidJob("not an object") + keys = set(data) + if not REQUIRED <= keys or keys - JOB_FIELDS: + raise InvalidJob("unexpected or missing fields") + if data["schema_version"] != 1 or isinstance(data["schema_version"], bool) or data["type"] != JOB_TYPE: + raise InvalidJob("unsupported schema_version/type") + try: + job_id = uuid.UUID(data["job_id"]) + attachment_id = uuid.UUID(data["attachment_id"]) + created = datetime.fromisoformat(str(data["created_at"]).replace("Z", "+00:00")) + except (ValueError, TypeError, AttributeError): + raise InvalidJob("bad uuid/date-time") from None + if created.tzinfo is None: + raise InvalidJob("created_at without timezone") + if "traceparent" in data and not (isinstance(data["traceparent"], str) and _TRACEPARENT.match(data["traceparent"])): + raise InvalidJob("bad traceparent") + if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512): + raise InvalidJob("bad tracestate") + return { + "job_id": job_id, + "attachment_id": attachment_id, + "pipeline_version": _int(data["pipeline_version"], 1), + "attempt": _int(data["attempt"], 1), + } + + +class Broker: + def __init__(self, url: Optional[str] = None): + self.url = url or config.BROKER_URL + self.connection: Optional[aio_pika.abc.AbstractRobustConnection] = None + self.channel: Optional[aio_pika.abc.AbstractChannel] = None + self.exchange = None + self.queue = None + + async def connect(self, prefetch: int = 0) -> None: + if not self.url: + raise RuntimeError("BROKER_URL or BROKER_URL_FILE is required") + self.connection = await aio_pika.connect_robust(self.url, timeout=5, client_properties={"connection_name": "messenger"}) + # publisher_confirms=True: publish() returns only after the broker took responsibility for the message. + self.channel = await self.connection.channel(publisher_confirms=True) + if prefetch: + await self.channel.set_qos(prefetch_count=prefetch) + await self.declare() + + async def declare(self) -> None: + ch = self.channel + dlx = await ch.declare_exchange(config.BROKER_DLX, ExchangeType.FANOUT, durable=True) + dlq = await ch.declare_queue(config.BROKER_DLQ, durable=True, arguments={"x-queue-type": "quorum"}) + await dlq.bind(dlx) + self.exchange = await ch.declare_exchange(config.BROKER_EXCHANGE, ExchangeType.DIRECT, durable=True) + self.queue = await ch.declare_queue(config.BROKER_QUEUE, durable=True, arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": config.BROKER_DLX, + "x-delivery-limit": config.BROKER_DELIVERY_LIMIT, + }) + await self.queue.bind(self.exchange, routing_key=config.BROKER_ROUTING_KEY) + + async def publish(self, payload: dict) -> None: + """Returns after the publisher confirm (basic.ack); raises on nack/return/connection loss.""" + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + await self.exchange.publish( + Message( + body, + content_type="application/json", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=payload["job_id"] + ":" + str(payload["attempt"]), + type=payload["type"], + ), + routing_key=config.BROKER_ROUTING_KEY, + mandatory=True, + timeout=10, + ) + + async def close(self) -> None: + if self.connection is not None: + await self.connection.close() diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..02db961c --- /dev/null +++ b/app/cache.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import time +from typing import Dict, Optional, Set, Union + +import redis.asyncio as redis +from redis.exceptions import RedisError + +from app import config + +logger = logging.getLogger("messenger.cache") + +REVOKED = "revoked" + + +class SessionCache: + def __init__(self, url: Optional[str]): + self._client: Optional[redis.Redis] = None + self._down_until = 0.0 + self._pending_revokes: Dict[bytes, float] = {} + self._retry_tasks: Set[asyncio.Task] = set() + if url: + pool = redis.BlockingConnectionPool.from_url( + url, + max_connections=config.CACHE_POOL_MAX, + timeout=config.CACHE_TIMEOUT, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ) + self._client = redis.Redis(connection_pool=pool) + + @property + def enabled(self) -> bool: + return self._client is not None + + def _usable(self) -> bool: + return self._client is not None and time.monotonic() >= self._down_until + + def _failed(self, op: str, exc: Exception) -> None: + self._down_until = time.monotonic() + config.CACHE_RETRY_AFTER + logger.warning("cache %s failed (%s); bypassing cache for %.0fs", op, type(exc).__name__, config.CACHE_RETRY_AFTER) + + @staticmethod + def _key(token_hash: bytes) -> str: + return "session:" + token_hash.hex() + + def _revoke_pending(self, token_hash: bytes) -> bool: + deadline = self._pending_revokes.get(token_hash) + if deadline is None: + return False + if time.monotonic() >= deadline: + self._pending_revokes.pop(token_hash, None) + return False + return True + + async def get(self, token_hash: bytes) -> Union[dict, str, None]: + if not self._usable() or self._revoke_pending(token_hash): + return None + try: + raw = await self._client.get(self._key(token_hash)) + except (RedisError, OSError) as exc: + self._failed("get", exc) + return None + if raw is None or raw == REVOKED: + return raw + try: + return json.loads(raw) + except ValueError: + return None + + async def put(self, token_hash: bytes, entry: dict, ttl_seconds: int) -> None: + ttl = min(ttl_seconds, config.SESSION_CACHE_MAX_SECONDS) + if ttl <= 0 or not self._usable(): + return + try: + await self._client.set(self._key(token_hash), json.dumps(entry), ex=ttl, nx=True) + except (RedisError, OSError) as exc: + self._failed("set", exc) + + async def _write_tombstone(self, token_hash: bytes, ttl_seconds: int) -> bool: + try: + await self._client.set(self._key(token_hash), REVOKED, ex=max(1, min(ttl_seconds, config.SESSION_CACHE_MAX_SECONDS))) + return True + except (RedisError, OSError) as exc: + self._failed("revoke", exc) + return False + + async def revoke(self, token_hash: bytes, ttl_seconds: int) -> bool: + if self._client is None: + return True + if await self._write_tombstone(token_hash, ttl_seconds): + return True + deadline = time.monotonic() + config.SESSION_CACHE_MAX_SECONDS + 1 + self._pending_revokes[token_hash] = deadline + task = asyncio.create_task(self._retry_revoke(token_hash, ttl_seconds, deadline)) + self._retry_tasks.add(task) + task.add_done_callback(self._retry_tasks.discard) + return False + + async def _retry_revoke(self, token_hash: bytes, ttl_seconds: int, deadline: float) -> None: + while time.monotonic() < deadline: + await asyncio.sleep(1) + remaining = int(deadline - time.monotonic()) + if remaining > 0 and await self._write_tombstone(token_hash, min(ttl_seconds, remaining)): + logger.info("revocation tombstone written after cache recovery") + break + self._pending_revokes.pop(token_hash, None) + + async def ping(self) -> bool: + if self._client is None: + return False + try: + return bool(await self._client.ping()) + except (RedisError, OSError): + return False + + async def close(self) -> None: + for task in list(self._retry_tasks): + task.cancel() + if self._client is not None: + await self._client.aclose() diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..ad5baf3c --- /dev/null +++ b/app/config.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +import secrets +from typing import Optional + + +def _secret(name: str) -> Optional[str]: + value = os.environ.get(name) + if value: + return value.strip() + path = os.environ.get(name + "_FILE") + if path and os.path.exists(path): + with open(path, encoding="utf-8") as fh: + return fh.read().strip() or None + return None + + +def _int(name: str, default: int) -> int: + return int(os.environ.get(name, default)) + + +def _float(name: str, default: float) -> float: + return float(os.environ.get(name, default)) + + +DATABASE_URL = _secret("DATABASE_URL") +CACHE_URL = _secret("CACHE_URL") + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:8333") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 5) +S3_POOL_MAX = _int("S3_POOL_MAX", 20) +# Without this cap 20 parallel 10 MiB PUTs drove SeaweedFS (512M limit) into an OOM kill. +S3_PUT_CONCURRENCY = _int("S3_PUT_CONCURRENCY", 4) + +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_MAX_SECONDS = _int("SESSION_CACHE_MAX_SECONDS", 300) + +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 3) +DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 3) +DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5) + +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.3) +CACHE_POOL_MAX = _int("CACHE_POOL_MAX", 20) +CACHE_RETRY_AFTER = _float("CACHE_RETRY_AFTER", 5) + +PASSWORD_HASH_CONCURRENCY = _int("PASSWORD_HASH_CONCURRENCY", 4) + +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1024 * 1024) +UPLOAD_TMP_DIR = os.environ.get("UPLOAD_TMP_DIR") or None +UPLOAD_TIMEOUT_SECONDS = _float("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 16) +DECODE_CONCURRENCY = _int("DECODE_CONCURRENCY", 1) + +ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) +SWEEP_INTERVAL_SECONDS = _int("SWEEP_INTERVAL_SECONDS", 60) + +INSTANCE_ID = "api-" + secrets.token_hex(4) +INSTANCE_ID_FILE = os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id") + +# --- lab5: broker, worker, retries --------------------------------------------------------------- +BROKER_URL = _secret("BROKER_URL") +BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments") +BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx") +BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq") +BROKER_ROUTING_KEY = "attachment.process.v1" +# Poison-message guard of the quorum queue: a message redelivered more often goes to the DLQ. +BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 10) + +PIPELINE_VERSION = _int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # = AMQP prefetch = parallel child processes +JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = _float("JOB_TIMEOUT_SECONDS", 60) # hard limit of one image transformation +LEASE_SECONDS = _float("LEASE_SECONDS", 20) # processing lease, renewed by heartbeat +HEARTBEAT_SECONDS = _float("HEARTBEAT_SECONDS", 5) +REAPER_INTERVAL_SECONDS = _float("REAPER_INTERVAL_SECONDS", 2) +STUCK_QUEUED_SECONDS = _float("STUCK_QUEUED_SECONDS", 300) # republish a queued job whose message vanished +RETRY_BASE_SECONDS = _float("RETRY_BASE_SECONDS", 2) +RETRY_MAX_SECONDS = _float("RETRY_MAX_SECONDS", 60) +WORKER_MEMORY_LIMIT_MB = _int("WORKER_MEMORY_LIMIT_MB", 768) # RLIMIT_AS of one transformation process +STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400) +RELAY_BATCH = _int("RELAY_BATCH", 100) +RELAY_POLL_SECONDS = _float("RELAY_POLL_SECONDS", 1) +FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1" diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..4d38adf8 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Dict, Optional + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(message: str, challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def unavailable() -> ApiError: + return ApiError(503, "unavailable", "service temporarily unavailable") + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 00000000..a9fdbb89 --- /dev/null +++ b/app/faults.py @@ -0,0 +1,67 @@ +"""Fault injection points for automated failure tests (track 5). Inert unless FAULT_INJECTION=1. + +A fault is armed by the operator CLI (python -m app.ops fault arm ...) as a row in fault_injections and is +consumed atomically by exactly one hit. Actions: + crash os._exit(137): the process dies like SIGKILL, nothing after this point runs (no ack, no COMMIT) + delay: sleep s seconds (widen a race window, e.g. let a newer pipeline version commit first) + transient raise a temporary failure -> retry with backoff + permanent raise a permanent failure -> failed without retries + +Points: worker.after_claim, worker.before_transform, worker.after_s3_before_commit, + worker.after_commit_before_ack, relay.after_confirm_before_mark +""" +from __future__ import annotations + +import asyncio +import logging +import os +import sys +import uuid +from typing import Optional + +from app import config + +logger = logging.getLogger("faults") + +POINTS = { + "worker.after_claim", + "worker.before_transform", + "worker.after_s3_before_commit", + "worker.after_commit_before_ack", + "relay.after_confirm_before_mark", +} + + +async def hit(pool, point: str, attachment_id: Optional[uuid.UUID], actor: str) -> None: + if not config.FAULT_INJECTION: + return + async with pool.acquire() as conn: + action = await conn.fetchval( + """ + UPDATE fault_injections SET remaining = remaining - 1 + WHERE id = (SELECT id FROM fault_injections + WHERE point = $1 AND (attachment_id IS NULL OR attachment_id = $2) AND remaining > 0 + ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED) + RETURNING action + """, + point, attachment_id, + ) + if action is None: + return + await conn.execute( + "INSERT INTO job_events (attachment_id, event, actor, detail) VALUES ($1, 'fault_injected', $2, $3)", + attachment_id, actor, point + " -> " + action, + ) + logger.warning("FAULT %s -> %s (attachment %s)", point, action, attachment_id) + if action == "crash": + sys.stdout.flush() + sys.stderr.flush() + os._exit(137) + if action.startswith("delay:"): + await asyncio.sleep(float(action.split(":", 1)[1])) + return + from app.imaging import ImageRejected, TransientFailure + if action == "transient": + raise TransientFailure("injected_transient") + if action == "permanent": + raise ImageRejected("processing_failed") diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..49d4c05f --- /dev/null +++ b/app/images.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import unicodedata +import warnings +from typing import BinaryIO, Optional, Tuple + +from PIL import Image + +from app import config +from app.errors import ApiError, payload_too_large, unsupported_media_type + +# Pillow itself refuses > 2x this value; our own check below rejects anything above it before decoding. +Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS +Image.LOAD_TRUNCATED_IMAGES = False + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +SNIFF_BYTES = 8 + +# Bidi overrides/isolates make "gpj.exe" look like "exe.jpg"; reject them together with control characters. +_BIDI_CONTROLS = set("‪‫‬‭‮⁦⁧⁨⁩‎‏") + + +def validate_filename(name: Optional[str]) -> str: + bad = ApiError(400, "invalid_request", "filename must be 1-255 characters without path separators or control characters") + if name is None or not 1 <= len(name) <= 255 or not name.strip() or name in (".", ".."): + raise bad + for ch in name: + if ch in "/\\" or ch in _BIDI_CONTROLS or unicodedata.category(ch) in ("Cc", "Zl", "Zp"): + raise bad + return name + + +def declared_type(header: Optional[str]) -> str: + media = (header or "").split(";", 1)[0].strip().lower() + if media not in FORMATS: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return media + + +def sniff(head: bytes) -> Optional[str]: + if head.startswith(PNG_MAGIC): + return "image/png" + if head.startswith(JPEG_MAGIC): + return "image/jpeg" + return None + + +def _open(fp: BinaryIO, fmt: str) -> Image.Image: + fp.seek(0) + return Image.open(fp, formats=[fmt]) + + +def inspect(fp: BinaryIO, content_type: str) -> Tuple[int, int]: + """Cheap synchronous checks before 202: format by content, dimensions from the header, pixel limit + before any decoding, PNG chunk CRCs (no IDAT inflation). Full decoding is the worker's job (lab5): a file + that passes these checks but cannot be decoded ends as status=failed, error_code=invalid_image. + """ + fmt = FORMATS[content_type] + corrupt = unsupported_media_type("file is not a valid " + fmt + " image") + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with _open(fp, fmt) as im: + width, height = im.size + if width < 1 or height < 1: + raise corrupt + if width * height > config.MAX_IMAGE_PIXELS: + raise payload_too_large("image exceeds %d pixels" % config.MAX_IMAGE_PIXELS) + if fmt == "PNG": + im.verify() # chunk structure and CRCs; the object is unusable afterwards + except ApiError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise payload_too_large("image exceeds %d pixels" % config.MAX_IMAGE_PIXELS) from None + except (OSError, SyntaxError, ValueError, EOFError, MemoryError): + raise corrupt from None + return width, height diff --git a/app/imaging.py b/app/imaging.py new file mode 100644 index 00000000..495c5b93 --- /dev/null +++ b/app/imaging.py @@ -0,0 +1,224 @@ +"""Versioned image pipeline, executed in an isolated child process with time and memory limits. + +Pipeline v1 (the main course version): + thumbnail JPEG q=85, fit into 256x256, no upscaling, floor rounding (min 1 px), EXIF orientation applied, + alpha composited on white -> 320x200 gives 256x160 + crop JPEG q=85, centred square 128x128 (small images are upscaled), same orientation/alpha rules + optimized WebP q=80 method=4, fit into 1280x1280 without upscaling, alpha kept +Metadata: no EXIF/XMP/ICC/comments are written (Pillow writes them only when passed explicitly), the +orientation is baked into the pixels instead. Encoding is deterministic for a given Pillow build, so a retry of +the same (attachment, version) produces byte-identical objects at the same keys. + +Pipeline v2 exists to demonstrate versioning (track 5): different quality settings, separate S3 keys. +""" +from __future__ import annotations + +import hashlib +import io +import multiprocessing +import os +import resource +import time +import warnings +from typing import Dict, List, Tuple + +from PIL import Image, ImageOps + +from app import config + +Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS +Image.LOAD_TRUNCATED_IMAGES = False + +PIPELINES: Dict[int, Dict[str, dict]] = { + 1: { + "thumbnail": {"box": 256, "format": "JPEG", "quality": 85}, + "crop": {"size": 128, "format": "JPEG", "quality": 85}, + "optimized": {"box": 1280, "format": "WEBP", "quality": 80, "method": 4}, + }, + 2: { + "thumbnail": {"box": 256, "format": "JPEG", "quality": 90}, + "crop": {"size": 128, "format": "JPEG", "quality": 90}, + "optimized": {"box": 1280, "format": "WEBP", "quality": 72, "method": 6}, + }, +} +MIME = {"JPEG": "image/jpeg", "WEBP": "image/webp", "PNG": "image/png"} +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} + + +class ImageRejected(Exception): + """Permanent: retrying the same bytes cannot succeed.""" + + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +class TransientFailure(Exception): + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +def fit_size(width: int, height: int, box: int) -> Tuple[int, int]: + """Fit into box x box keeping proportions, never upscale, round down, at least 1 px.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def _flatten(im: Image.Image) -> Image.Image: + """RGB for JPEG: alpha (RGBA/LA/P+transparency) composited on white.""" + if im.mode in ("I", "I;16", "I;16B", "I;16L", "F"): + im = im.convert("I").point(lambda v: v * (1 / 256)).convert("L") + if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info): + rgba = im.convert("RGBA") + out = Image.new("RGB", rgba.size, (255, 255, 255)) + out.paste(rgba, mask=rgba.getchannel("A")) + return out + return im.convert("RGB") + + +def _with_alpha(im: Image.Image) -> Image.Image: + if im.mode in ("I", "I;16", "I;16B", "I;16L", "F"): + im = im.convert("I").point(lambda v: v * (1 / 256)).convert("L") + if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info): + return im.convert("RGBA") + return im.convert("RGB") + + +def _encode(im: Image.Image, spec: dict) -> bytes: + buf = io.BytesIO() + if spec["format"] == "JPEG": + im.save(buf, "JPEG", quality=spec["quality"], optimize=True, progressive=False, subsampling="4:2:0") + else: + im.save(buf, "WEBP", quality=spec["quality"], method=spec["method"], exact=False) + return buf.getvalue() + + +def transform(src_path: str, content_type: str, out_dir: str, version: int) -> List[dict]: + """Decode once, produce every variant of `version`. Runs inside the limited child process.""" + if version not in PIPELINES: + raise ImageRejected("unsupported_pipeline") + spec = PIPELINES[version] + fmt = FORMATS.get(content_type) + if fmt is None: + raise ImageRejected("invalid_image") + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(src_path, formats=[fmt]) as im: + width, height = im.size + if width < 1 or height < 1: + raise ImageRejected("invalid_image") + if width * height > config.MAX_IMAGE_PIXELS: + raise ImageRejected("image_too_large") + orientation = im.getexif().get(0x0112, 1) + if orientation in (5, 6, 7, 8): + width, height = height, width + largest = spec["optimized"]["box"] + if fmt == "JPEG": + # DCT-domain downscale while decoding: memory ~ pixels/4..1/64 for big photos. + im.draft("RGB", (min(width, largest), min(height, largest))) + im.load() + im = ImageOps.exif_transpose(im) + except ImageRejected: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise ImageRejected("image_too_large") from None + except MemoryError: + raise ImageRejected("image_too_large") from None + except (OSError, SyntaxError, ValueError, EOFError, TypeError, IndexError): + raise ImageRejected("invalid_image") from None + + flat = _flatten(im) + outputs = [] + + t = spec["thumbnail"] + tw, th = fit_size(width, height, t["box"]) + outputs.append(("thumbnail", flat.resize((tw, th), Image.LANCZOS, reducing_gap=3.0), t)) + + c = spec["crop"] + outputs.append(("crop", ImageOps.fit(flat, (c["size"], c["size"]), Image.LANCZOS, centering=(0.5, 0.5)), c)) + + o = spec["optimized"] + ow, oh = fit_size(width, height, o["box"]) + rich = _with_alpha(im) + outputs.append(("optimized", rich if rich.size == (ow, oh) else rich.resize((ow, oh), Image.LANCZOS, reducing_gap=3.0), o)) + + result = [] + for name, img, s in outputs: + data = _encode(img, s) + path = os.path.join(out_dir, name) + with open(path, "wb") as fh: + fh.write(data) + result.append({ + "name": name, + "content_type": MIME[s["format"]], + "path": path, + "size_bytes": len(data), + "width": img.size[0], + "height": img.size[1], + "sha256": hashlib.sha256(data).hexdigest(), + }) + return result + + +def _child(conn, src_path: str, content_type: str, out_dir: str, version: int, memory_mb: int) -> None: + limit = memory_mb * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + try: + conn.send(("ok", transform(src_path, content_type, out_dir, version))) + except ImageRejected as exc: + conn.send(("rejected", exc.code)) + except MemoryError: + conn.send(("rejected", "image_too_large")) + except Exception as exc: # unexpected: transient, bounded by max attempts + conn.send(("transient", "processing_failed:" + type(exc).__name__)) + finally: + conn.close() + + +_ctx = None + + +def _context(): + global _ctx + if _ctx is None: + # forkserver: children are forked from a clean single-threaded server (not from the asyncio/boto3 parent). + _ctx = multiprocessing.get_context("forkserver") + _ctx.set_forkserver_preload(["app.imaging"]) + return _ctx + + +def run_isolated(src_path: str, content_type: str, out_dir: str, version: int, + timeout: float, memory_mb: int) -> List[dict]: + """Blocking; call from a thread. Kills the child on timeout (hung decoder) and reports how it ended.""" + ctx = _context() + parent, child = ctx.Pipe(duplex=False) + proc = ctx.Process(target=_child, args=(child, src_path, content_type, out_dir, version, memory_mb), daemon=True) + proc.start() + child.close() + try: + deadline = time.monotonic() + timeout + if not parent.poll(max(0.0, deadline - time.monotonic())): + proc.kill() + proc.join(5) + raise TransientFailure("processing_timeout") + try: + kind, value = parent.recv() + except EOFError: + proc.join(5) + # killed by the kernel/cgroup (e.g. OOM) or crashed in native code + raise TransientFailure("worker_process_died:%s" % proc.exitcode) from None + proc.join(5) + finally: + parent.close() + if proc.is_alive(): + proc.kill() + if kind == "ok": + return value + if kind == "rejected": + raise ImageRejected(value) + raise TransientFailure(value) diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 00000000..443ddb07 --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,103 @@ +"""Job state machine in PostgreSQL (source of truth). The broker only wakes workers up. + +Every state change that needs a message writes an outbox row in the SAME transaction (transactional outbox), +so there is no window where a job is queued in the DB without a pending publication. +""" +from __future__ import annotations + +import json +import random +import uuid +from typing import Optional + +import asyncpg + +from app import config +from app.broker import job_payload + + +def backoff_seconds(attempt: int) -> float: + """Exponential backoff with 'equal jitter': half fixed, half random. attempt = failed attempt number (1..).""" + ceiling = min(config.RETRY_MAX_SECONDS, config.RETRY_BASE_SECONDS * (2 ** max(0, attempt - 1))) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +async def event(conn: asyncpg.Connection, job_id: Optional[uuid.UUID], attachment_id: Optional[uuid.UUID], + name: str, actor: str, attempt: Optional[int] = None, detail: Optional[str] = None) -> None: + await conn.execute( + "INSERT INTO job_events (job_id, attachment_id, event, attempt, actor, detail) VALUES ($1, $2, $3, $4, $5, $6)", + job_id, attachment_id, name, attempt, actor, detail, + ) + + +async def enqueue(conn: asyncpg.Connection, job_id: uuid.UUID, attachment_id: uuid.UUID, pipeline_version: int, + attempt: int, delay: float = 0.0) -> None: + """Outbox row for the given attempt; must run inside the transaction that made the job 'queued'.""" + payload = job_payload(job_id, attachment_id, pipeline_version, attempt) + await conn.execute( + """ + INSERT INTO outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, clock_timestamp() + make_interval(secs => $3)) + """, + job_id, json.dumps(payload), float(delay), + ) + if delay <= 0: + await conn.execute("SELECT pg_notify('outbox', '')") + + +async def create_job(conn: asyncpg.Connection, attachment_id: uuid.UUID, pipeline_version: int, actor: str) -> uuid.UUID: + job_id = uuid.uuid4() + await conn.execute( + """ + INSERT INTO jobs (job_id, attachment_id, pipeline_version, max_attempts) + VALUES ($1, $2, $3, $4) + """, + job_id, attachment_id, pipeline_version, config.JOB_MAX_ATTEMPTS, + ) + await enqueue(conn, job_id, attachment_id, pipeline_version, 1) + await event(conn, job_id, attachment_id, "accepted", actor, 0, "pipeline_version=%d" % pipeline_version) + return job_id + + +async def fail(conn: asyncpg.Connection, job: asyncpg.Record, code: str, last_error: str, actor: str) -> None: + """processing -> failed. The attachment fails only if no version has been published yet.""" + await conn.execute( + """ + UPDATE jobs SET status = 'failed', lease_owner = NULL, lease_until = NULL, error_code = $2, + last_error = $3, updated_at = clock_timestamp() + WHERE job_id = $1 + """, + job["job_id"], code, last_error[:200], + ) + await conn.execute( + """ + UPDATE attachments SET status = 'failed', error_code = $2, updated_at = clock_timestamp() + WHERE id = $1 AND status IN ('queued', 'processing') + """, + job["attachment_id"], code, + ) + await event(conn, job["job_id"], job["attachment_id"], "failed", actor, job["attempt"], code + " (" + last_error[:120] + ")") + + +async def retry_or_fail(conn: asyncpg.Connection, job: asyncpg.Record, reason: str, actor: str) -> str: + """processing -> queued (with backoff) while attempts remain, otherwise -> failed retry_exhausted.""" + if job["attempt"] >= job["max_attempts"]: + await fail(conn, job, "retry_exhausted", reason, actor) + return "failed" + delay = backoff_seconds(job["attempt"]) + await conn.execute( + """ + UPDATE jobs SET status = 'queued', lease_owner = NULL, lease_until = NULL, last_error = $2, + next_attempt_at = clock_timestamp() + make_interval(secs => $3), updated_at = clock_timestamp() + WHERE job_id = $1 + """, + job["job_id"], reason[:200], delay, + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = $1 AND status = 'processing'", + job["attachment_id"], + ) + await enqueue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1, delay) + await event(conn, job["job_id"], job["attachment_id"], "retry_scheduled", actor, job["attempt"], + "%s; next attempt %d in %.2fs" % (reason[:120], job["attempt"] + 1, delay)) + return "queued" diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..9c2a824a --- /dev/null +++ b/app/main.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import re +import tempfile +import uuid +from contextlib import asynccontextmanager +from typing import Optional + +import asyncpg +from fastapi import Depends, FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect + +from app import auth, config, images +from app.cache import SessionCache +from app.errors import ApiError, payload_too_large, unsupported_media_type +from app.models import (Attachment, Chat, ChatPage, CreateChat, CreateMessage, CreateUser, Health, Message, MessagePage, + Session, User) +from app.objects import ObjectStore, S3Unavailable, original_key, staging_key, storage_unavailable +from app.storage import Store, attachment_row + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("messenger") + +store = Store() +cache = SessionCache(config.CACHE_URL) +objects = ObjectStore() +upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) +inspect_slots = asyncio.Semaphore(config.DECODE_CONCURRENCY) +s3_put_slots = asyncio.Semaphore(config.S3_PUT_CONCURRENCY) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + if not config.DATABASE_URL: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required") + if not (config.S3_ACCESS_KEY_ID and config.S3_SECRET_ACCESS_KEY): + raise RuntimeError("S3_ACCESS_KEY_ID(_FILE) and S3_SECRET_ACCESS_KEY(_FILE) are required") + await store.open() + logger.info("session cache: %s", "valkey" if cache.enabled else "disabled") + logger.info("object storage: %s bucket=%s", config.S3_ENDPOINT, config.S3_BUCKET) + try: + with open(config.INSTANCE_ID_FILE, "w", encoding="ascii") as fh: + fh.write(config.INSTANCE_ID + "\n") + except OSError: + logger.warning("cannot write %s", config.INSTANCE_ID_FILE) + logger.info("instance id: %s", config.INSTANCE_ID) + yield + await cache.close() + await store.close() + + +app = FastAPI(title="MTUSI Messenger — lab5", lifespan=lifespan) + +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +DB_UNAVAILABLE = ( + OSError, + asyncpg.PostgresConnectionError, + asyncpg.InterfaceError, + asyncpg.TooManyConnectionsError, + asyncpg.exceptions.OperatorInterventionError, + asyncpg.exceptions.QueryCanceledError, +) + + +def error_body(code: str, message: str, request_id: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def error_response(request: Request, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") + return JSONResponse(status_code=status, content=error_body(code, message, request_id), headers=headers) + + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request_id = uuid.uuid4().hex + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logger.exception("unhandled error for request_id=%s", request_id) + response = JSONResponse(status_code=500, content=error_body("internal_error", "internal error", request_id)) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.status_code, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "request failed validation") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + code = {404: "not_found", 405: "invalid_request", 413: "payload_too_large", 415: "unsupported_media_type"}.get( + exc.status_code, "invalid_request") + message = exc.detail if isinstance(exc.detail, str) else "request failed" + return error_response(request, exc.status_code, code, message) + + +async def db_unavailable_handler(request: Request, exc: Exception): + logger.warning("database unavailable for request_id=%s: %s", request.state.request_id, type(exc).__name__) + return error_response(request, 503, "unavailable", "service temporarily unavailable") + + +for _exc in DB_UNAVAILABLE: + app.add_exception_handler(_exc, db_unavailable_handler) + + +@app.exception_handler(S3Unavailable) +async def s3_unavailable_handler(request: Request, exc: S3Unavailable): + err = storage_unavailable() + return error_response(request, err.status_code, err.code, err.message) + + +async def current_user(request: Request) -> dict: + return await auth.authenticate(store, cache, request.headers.get("Authorization")) + + +def parse_uuid(value: str, field: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except ValueError: + raise ApiError(400, "invalid_request", f"{field} is not a valid UUID") from None + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "chat not found") + + +@app.get("/health/live", response_model=Health) +async def liveness(): + return Health() + + +@app.get("/health/ready", response_model=Health) +async def readiness(): + try: + await store.ping() + except DB_UNAVAILABLE: + raise ApiError(503, "unavailable", "database is not available") from None + return Health() + + +@app.post("/api/v1/users", response_model=User, status_code=201) +async def register_user(body: CreateUser): + if not body.display_name.strip() or not body.password.strip(): + raise ApiError(400, "invalid_request", "display_name and password must not be blank") + password_hash = await auth.hash_password(body.password) + return await store.create_user(body.username, body.display_name, password_hash) + + +@app.get("/api/v1/users/me", response_model=User) +async def current_user_endpoint(user: dict = Depends(current_user)): + return user + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +async def create_session(request: Request): + return await auth.login(store, request.headers.get("Authorization")) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def delete_session(request: Request): + await auth.logout(store, cache, request.headers.get("Authorization")) + return Response(status_code=204) + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +async def create_chat(body: CreateChat, user: dict = Depends(current_user)): + if not body.title.strip(): + raise ApiError(400, "invalid_request", "title must not be blank") + if len(set(body.member_ids)) != len(body.member_ids): + raise ApiError(400, "invalid_request", "member_ids must not contain duplicates") + if uuid.UUID(user["id"]) not in body.member_ids: + raise ApiError(400, "invalid_request", "caller must be a member of the chat") + return await store.create_chat(body.title, list(body.member_ids)) + + +@app.get("/api/v1/chats", response_model=ChatPage) +async def list_chats( + user: dict = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +): + return await store.list_chats(uuid.UUID(user["id"]), limit, cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +async def get_chat(chat_id: str, user: dict = Depends(current_user)): + chat = await store.get_chat_for_member(parse_uuid(chat_id, "chat_id"), uuid.UUID(user["id"])) + if chat is None: + raise not_found() + return chat + + +@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: str, + body: CreateMessage, + user: dict = Depends(current_user), + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), +): + chat_uuid = parse_uuid(chat_id, "chat_id") + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}") + if len(set(body.attachment_ids)) != len(body.attachment_ids): + raise ApiError(400, "invalid_request", "attachment_ids must be unique") + if not body.text.strip() and not body.attachment_ids: + if not await store.is_member(chat_uuid, uuid.UUID(user["id"])): + raise not_found() + raise ApiError(400, "invalid_request", "text must not be blank without attachments") + canonical = json.dumps(body.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + request_hash = hashlib.sha256(canonical.encode("utf-8")).digest() + message = await store.create_message( + chat_uuid, uuid.UUID(user["id"]), body.text, list(body.attachment_ids), idempotency_key, request_hash) + if message is None: + raise not_found() + return message + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +async def list_messages( + chat_id: str, + user: dict = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +): + chat_uuid = parse_uuid(chat_id, "chat_id") + if not await store.is_member(chat_uuid, uuid.UUID(user["id"])): + raise not_found() + return await store.list_messages(chat_uuid, limit, cursor) + + +VARIANTS = {"original", "thumbnail", "crop", "optimized"} + + +async def receive_body(request: Request, spool) -> tuple: + """Stream the body into a spooled temp file: RAM holds at most UPLOAD_SPOOL_BYTES per upload.""" + digest = hashlib.sha256() + size = 0 + head = b"" + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise payload_too_large("file exceeds %d bytes" % config.MAX_UPLOAD_BYTES) + if len(head) < images.SNIFF_BYTES: + head += chunk[:images.SNIFF_BYTES - len(head)] + digest.update(chunk) + spool.write(chunk) + return size, digest.hexdigest(), head + + +@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=202) +async def upload_attachment( + chat_id: str, + request: Request, + user: dict = Depends(current_user), + filename: Optional[str] = Query(default=None), +): + chat_uuid = parse_uuid(chat_id, "chat_id") + owner_id = uuid.UUID(user["id"]) + if not await store.is_member(chat_uuid, owner_id): + raise not_found() + filename = images.validate_filename(filename) + declared = images.declared_type(request.headers.get("Content-Type")) + length = request.headers.get("Content-Length") + if length is not None: + if not length.isdigit(): + raise ApiError(400, "invalid_request", "invalid Content-Length") + if int(length) > config.MAX_UPLOAD_BYTES: + raise payload_too_large("file exceeds %d bytes" % config.MAX_UPLOAD_BYTES) + + async with upload_slots: + spool = tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES, dir=config.UPLOAD_TMP_DIR) + try: + try: + size, sha256, head = await asyncio.wait_for(receive_body(request, spool), config.UPLOAD_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + raise ApiError(400, "invalid_request", "upload was not completed in time") from None + except ClientDisconnect: + logger.info("upload aborted by client after partial body; nothing stored") + raise ApiError(400, "invalid_request", "upload interrupted") from None + if size == 0: + raise unsupported_media_type("empty body is not an image") + detected = images.sniff(head) + if detected is None: + raise unsupported_media_type("content is neither PNG nor JPEG") + if detected != declared: + raise unsupported_media_type("content does not match Content-Type " + declared) + async with inspect_slots: + width, height = await run_in_threadpool(images.inspect, spool, detected) + + attachment_id = uuid.uuid4() + stage = staging_key(attachment_id) + # 1) full verified original -> durable staging (S3 checks SHA-256 of the received bytes) + async with s3_put_slots: + await objects.put(stage, spool, size, detected, sha256) + # 2) ONE transaction: attachment(queued) + job + outbox. Only after COMMIT do we answer 202. + # A crash between 1) and 2) leaves an unreferenced staging object -> sweeper after the grace period. + try: + accepted = await store.create_queued_attachment( + attachment_id, chat_uuid, owner_id, filename, detected, size, sha256, width, height, + stage, original_key(attachment_id), config.INSTANCE_ID, + ) + except DB_UNAVAILABLE: + await objects.delete_quietly(stage) + raise + if accepted is None: + await objects.delete_quietly(stage) + raise not_found() + return JSONResponse(status_code=202, content=accepted) + finally: + spool.close() + + +async def readable_attachment(attachment_id: str, user: dict): + row, variants = await store.get_attachment_for_member(parse_uuid(attachment_id, "attachment_id"), uuid.UUID(user["id"])) + if row is None: + raise ApiError(404, "not_found", "attachment not found") + return row, variants + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +async def get_attachment(attachment_id: str, user: dict = Depends(current_user)): + row, variants = await readable_attachment(attachment_id, user) + return attachment_row(row, variants) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: str, + user: dict = Depends(current_user), + variant: str = Query(default="original"), +): + if variant not in VARIANTS: + raise ApiError(400, "invalid_request", "unknown variant") + row, variants = await readable_attachment(attachment_id, user) + if row["status"] != "ready": + raise ApiError(409, "conflict", "attachment is %s, content is not available yet" % row["status"]) + if variant == "original": + key, ctype, expected = row["object_key"], row["content_type"], row["size_bytes"] + else: + found = [v for v in variants if v["name"] == variant] + if not found: + raise ApiError(404, "not_found", "variant not found") + key, ctype, expected = found[0]["object_key"], found[0]["content_type"], found[0]["size_bytes"] + chunks, length, close = await objects.open(key) + if length != expected: + close() + logger.error("object size differs from metadata for attachment %s variant %s", row["id"], variant) + raise storage_unavailable() + return StreamingResponse( + chunks, + media_type=ctype, + headers={ + "Content-Length": str(length), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + background=BackgroundTask(close), + ) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..29df5991 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +from pathlib import Path + +import asyncpg + +from app import config + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("migrate") + +MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / "migrations" +LOCK_ID = 2026_0002 + + +async def connect_with_retry(attempts: int = 30) -> asyncpg.Connection: + for attempt in range(1, attempts + 1): + try: + return await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT) + except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError) as exc: + logger.warning("database not reachable (attempt %d/%d): %s", attempt, attempts, type(exc).__name__) + await asyncio.sleep(1) + raise SystemExit("database is not reachable") + + +async def migrate() -> int: + if not config.DATABASE_URL: + raise SystemExit("DATABASE_URL or DATABASE_URL_FILE is required") + conn = await connect_with_retry() + applied_now = 0 + try: + await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID) + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + done = {r["version"] for r in await conn.fetch("SELECT version FROM schema_migrations")} + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + version = path.stem + if version in done: + continue + logger.info("applying %s", version) + async with conn.transaction(): + await conn.execute(path.read_text(encoding="utf-8")) + await conn.execute("INSERT INTO schema_migrations (version) VALUES ($1)", version) + applied_now += 1 + logger.info("migrations up to date: %d applied now, %d before", applied_now, len(done)) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID) + await conn.close() + return applied_now + + +if __name__ == "__main__": + asyncio.run(migrate()) + sys.exit(0) diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..d1247bb2 --- /dev/null +++ b/app/models.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import uuid +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field + +USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$" + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + +class User(BaseModel): + id: uuid.UUID + username: str + display_name: str + created_at: str + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: List[uuid.UUID] = Field(min_length=1, max_length=100) + + +class Chat(BaseModel): + id: uuid.UUID + title: str + member_ids: List[uuid.UUID] + created_at: str + + +class ChatPage(BaseModel): + items: List[Chat] + next_cursor: Optional[str] = None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: List[uuid.UUID] = Field(default_factory=list, max_length=10) + + +class Message(BaseModel): + id: uuid.UUID + chat_id: uuid.UUID + sender_id: uuid.UUID + text: str + created_at: str + attachment_ids: List[uuid.UUID] + + +class MessagePage(BaseModel): + items: List[Message] + next_cursor: Optional[str] = None + + +class Session(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: User + + +class Variant(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: uuid.UUID + chat_id: uuid.UUID + owner_id: uuid.UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: List[Variant] + error_code: Optional[str] + created_at: str + updated_at: str + + +class Health(BaseModel): + status: str = "ok" diff --git a/app/objects.py b/app/objects.py new file mode 100644 index 00000000..a6033e86 --- /dev/null +++ b/app/objects.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import base64 +import logging +import uuid +from datetime import datetime +from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from starlette.concurrency import run_in_threadpool + +from app import config +from app.errors import ApiError + +logger = logging.getLogger("messenger.s3") + +# Private prefixes of the single private bucket: +# staging/ full original accepted by the API before 202 (durable handoff) +# attachments//original published original (same bytes, written by the worker) +# attachments//v/.* variants of pipeline version N; key = effect key, so a retry +# overwrites the same object instead of creating a second set +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "attachments/" +DOWNLOAD_CHUNK = 64 * 1024 +VARIANT_EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"} + + +class S3Unavailable(Exception): + pass + + +class S3NotFound(Exception): + pass + + +def staging_key(attachment_id: uuid.UUID) -> str: + return STAGING_PREFIX + attachment_id.hex + + +def original_key(attachment_id: uuid.UUID) -> str: + return "%s%s/original" % (FINAL_PREFIX, attachment_id.hex) + + +def variant_key(attachment_id: uuid.UUID, pipeline_version: int, name: str, content_type: str) -> str: + return "%s%s/v%d/%s.%s" % (FINAL_PREFIX, attachment_id.hex, pipeline_version, name, VARIANT_EXT[content_type]) + + +def _client(access_key: Optional[str], secret_key: Optional[str]): + cfg = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=config.S3_CONNECT_TIMEOUT, + read_timeout=config.S3_READ_TIMEOUT, + retries={"max_attempts": 1, "mode": "standard"}, + max_pool_connections=config.S3_POOL_MAX, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ) + return boto3.session.Session().client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=cfg, + ) + + +def _code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "") + + +class ObjectStore: + """Synchronous boto3 client; async wrappers run it in the threadpool.""" + + def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None): + self.bucket = config.S3_BUCKET + self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY) + + def put_sync(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + fileobj.seek(0) + try: + # x-amz-checksum-sha256: S3 rejects the PUT if the received bytes differ from what we hashed. + self._s3.put_object( + Bucket=self.bucket, + Key=key, + Body=fileobj, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "") + raise S3Unavailable() from None + + async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + await run_in_threadpool(self.put_sync, key, fileobj, size, content_type, sha256_hex) + + def download_sync(self, key: str, fileobj: BinaryIO, max_bytes: int) -> int: + """Copy an object into fileobj; raises S3NotFound for a missing key (permanent), S3Unavailable otherwise.""" + try: + obj = self._s3.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if _code(exc) in ("NoSuchKey", "404", "NotFound"): + raise S3NotFound(key) from None + raise S3Unavailable() from None + except BotoCoreError: + raise S3Unavailable() from None + size = 0 + try: + for chunk in obj["Body"].iter_chunks(DOWNLOAD_CHUNK): + size += len(chunk) + if size > max_bytes: + raise S3Unavailable() + fileobj.write(chunk) + except BotoCoreError: + raise S3Unavailable() from None + finally: + obj["Body"].close() + return size + + def _open(self, key: str): + try: + return self._s3.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + logger.error("s3 get failed for existing metadata: %s", _code(exc)) + raise S3Unavailable() from None + except BotoCoreError as exc: + logger.warning("s3 get failed: %s", type(exc).__name__) + raise S3Unavailable() from None + + async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]: + obj = await run_in_threadpool(self._open, key) + body = obj["Body"] + return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close + + def delete_sync(self, key: str) -> None: + try: + self._s3.delete_object(Bucket=self.bucket, Key=key) + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable() from exc + + async def delete_quietly(self, key: str) -> bool: + try: + await run_in_threadpool(self.delete_sync, key) + return True + except S3Unavailable: + logger.warning("compensating delete failed; sweeper will retry later") + return False + + def list_prefix(self, prefix: str) -> Iterator[Tuple[str, datetime]]: + paginator = self._s3.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix, PaginationConfig={"PageSize": 1000}): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"] + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable() from exc + + def delete_many(self, keys: List[str]) -> int: + deleted = 0 + for i in range(0, len(keys), 1000): + batch = keys[i:i + 1000] + try: + out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True}) + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable() from exc + deleted += len(batch) - len(out.get("Errors", [])) + return deleted + + def create_bucket(self) -> bool: + try: + self._s3.create_bucket(Bucket=self.bucket) + return True + except ClientError as exc: + if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + return False + raise + + +def storage_unavailable() -> ApiError: + return ApiError(503, "unavailable", "attachment storage temporarily unavailable") diff --git a/app/ops.py b/app/ops.py new file mode 100644 index 00000000..2d014e9f --- /dev/null +++ b/app/ops.py @@ -0,0 +1,253 @@ +"""Operator CLI (runs inside the worker image, never exposed over HTTP). + + python -m app.ops failed [--limit N] registry of failed jobs (the "DLQ" of the DB) + python -m app.ops replay --actor NAME explicit failed -> queued, audited + python -m app.ops reprocess --pipeline-version N new job for a newer pipeline version + python -m app.ops inspect JSON: attachment, jobs, variants, events, S3 keys + python -m app.ops stats queue depth, DLQ, outbox backlog, job states + python -m app.ops fault arm [--attachment ID] [--count N] + python -m app.ops fault clear + python -m app.ops republish [--attempt N] [--pipeline-version N] + raw publish bypassing the outbox: duplicate / stale / reordered message for fault tests + python -m app.ops dlq-peek show (and requeue nothing from) the broker DLQ +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import uuid + +import asyncpg + +from app import config, faults, jobs +from app.broker import Broker, job_payload +from app.objects import FINAL_PREFIX, ObjectStore + +ACTOR_PREFIX = "operator:" + + +def _ts(v): + return v.isoformat() if v is not None else None + + +async def connect(): + return await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT) + + +async def cmd_failed(args) -> int: + conn = await connect() + rows = await conn.fetch( + """ + SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, j.last_error, + j.replays, j.updated_at, a.staging_deleted_at IS NULL AS staging_present + FROM jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT $1 + """, args.limit) + print(json.dumps([{k: (str(v) if isinstance(v, uuid.UUID) else _ts(v) if hasattr(v, "isoformat") else v) + for k, v in dict(r).items()} for r in rows], ensure_ascii=False, indent=1)) + await conn.close() + return 0 + + +async def cmd_replay(args) -> int: + """failed -> queued. Refuses anything that is not failed: a ready job is never executed again.""" + if not args.actor: + print("--actor is required for the audit log", file=sys.stderr) + return 2 + conn = await connect() + try: + async with conn.transaction(): + job = await conn.fetchrow( + """ + SELECT j.*, a.status AS a_status, a.staging_deleted_at FROM jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = $1 ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a + """, uuid.UUID(args.attachment_id)) + if job is None: + print("no job for this attachment", file=sys.stderr) + return 1 + if job["status"] != "failed": + await jobs.event(conn, job["job_id"], job["attachment_id"], "replay_refused", ACTOR_PREFIX + args.actor, + job["attempt"], "status=%s" % job["status"]) + print("refused: job is %s (attachment %s); replay applies only to failed jobs" % (job["status"], job["a_status"])) + return 3 + if job["staging_deleted_at"] is not None and job["a_status"] != "ready": + print("refused: staging object already removed by retention; upload the file again", file=sys.stderr) + return 4 + await conn.execute( + """ + UPDATE jobs SET status = 'queued', max_attempts = attempt + $2, replays = replays + 1, + error_code = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = $1 + """, job["job_id"], config.JOB_MAX_ATTEMPTS) + await conn.execute( + """ + UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND status = 'failed' + """, job["attachment_id"]) + await jobs.enqueue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1) + await jobs.event(conn, job["job_id"], job["attachment_id"], "replay", ACTOR_PREFIX + args.actor, job["attempt"], + "previous error %s; %d more attempts" % (job["error_code"], config.JOB_MAX_ATTEMPTS)) + print("replayed job %s (attempt %d next)" % (job["job_id"], job["attempt"] + 1)) + return 0 + finally: + await conn.close() + + +async def cmd_reprocess(args) -> int: + conn = await connect() + try: + async with conn.transaction(): + att = await conn.fetchrow("SELECT status, pipeline_version FROM attachments WHERE id = $1 FOR UPDATE", + uuid.UUID(args.attachment_id)) + if att is None: + print("unknown attachment", file=sys.stderr) + return 1 + if att["pipeline_version"] is not None and att["pipeline_version"] >= args.pipeline_version: + print("refused: v%d already published" % att["pipeline_version"]) + return 3 + try: + async with conn.transaction(): + job_id = await jobs.create_job(conn, uuid.UUID(args.attachment_id), args.pipeline_version, + ACTOR_PREFIX + args.actor) + except asyncpg.UniqueViolationError: + print("refused: a job for this (attachment, pipeline_version) already exists") + return 3 + print("created job %s for v%d" % (job_id, args.pipeline_version)) + return 0 + finally: + await conn.close() + + +async def inspect(conn, attachment_id: uuid.UUID) -> dict: + att = await conn.fetchrow("SELECT * FROM attachments WHERE id = $1", attachment_id) + if att is None: + return {"attachment": None} + job_rows = await conn.fetch("SELECT * FROM jobs WHERE attachment_id = $1 ORDER BY pipeline_version", attachment_id) + variants = await conn.fetch( + "SELECT pipeline_version, name, object_key, size_bytes, width, height, sha256, attempt FROM attachment_variants " + "WHERE attachment_id = $1 ORDER BY pipeline_version, name", attachment_id) + events = await conn.fetch( + "SELECT event, attempt, actor, detail, created_at FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id) + outbox = await conn.fetch( + "SELECT o.id, o.payload->>'attempt' AS attempt, o.published_at IS NOT NULL AS published, o.publish_count " + "FROM outbox o JOIN jobs j ON j.job_id = o.job_id WHERE j.attachment_id = $1 ORDER BY o.id", attachment_id) + store = ObjectStore() + keys = await asyncio.to_thread(lambda: sorted(k for k, _ in store.list_prefix(FINAL_PREFIX + attachment_id.hex + "/"))) + staging = await asyncio.to_thread(lambda: [k for k, _ in store.list_prefix(att["staging_key"] or "-")]) + + def clean(row): + return {k: (str(v) if isinstance(v, uuid.UUID) else _ts(v) if hasattr(v, "isoformat") else v) for k, v in dict(row).items()} + return { + "attachment": {k: v for k, v in clean(att).items() if k in ( + "id", "status", "error_code", "pipeline_version", "sha256", "size_bytes", "staging_deleted_at", "updated_at")}, + "jobs": [{k: v for k, v in clean(j).items() if k in ( + "job_id", "pipeline_version", "status", "attempt", "max_attempts", "error_code", "last_error", "replays", + "lease_owner")} for j in job_rows], + "variants": [clean(v) for v in variants], + "events": [clean(e) for e in events], + "outbox": [clean(o) for o in outbox], + "s3_final_keys": keys, + "s3_staging_keys": staging, + } + + +async def cmd_inspect(args) -> int: + conn = await connect() + print(json.dumps(await inspect(conn, uuid.UUID(args.attachment_id)), ensure_ascii=False, indent=1)) + await conn.close() + return 0 + + +async def cmd_stats(args) -> int: + conn = await connect() + states = {r["status"]: r["n"] for r in await conn.fetch("SELECT status, count(*) AS n FROM jobs GROUP BY status")} + backlog = await conn.fetchval("SELECT count(*) FROM outbox WHERE published_at IS NULL") + await conn.close() + broker = Broker() + await broker.connect() + q = await broker.channel.declare_queue(config.BROKER_QUEUE, passive=True) + d = await broker.channel.declare_queue(config.BROKER_DLQ, passive=True) + out = {"jobs": states, "outbox_unpublished": backlog, + "queue_ready_messages": q.declaration_result.message_count, + "queue_consumers": q.declaration_result.consumer_count, + "dlq_messages": d.declaration_result.message_count} + await broker.close() + print(json.dumps(out)) + return 0 + + +async def cmd_fault(args) -> int: + conn = await connect() + if args.fault_cmd == "clear": + n = await conn.execute("DELETE FROM fault_injections") + print("cleared", n.split()[-1]) + elif args.fault_cmd == "arm": + if args.point not in faults.POINTS: + print("unknown point; one of: " + ", ".join(sorted(faults.POINTS)), file=sys.stderr) + return 2 + if not (args.action in ("crash", "transient", "permanent") or args.action.startswith("delay:")): + print("action: crash | transient | permanent | delay:", file=sys.stderr) + return 2 + await conn.execute("INSERT INTO fault_injections (point, action, attachment_id, remaining) VALUES ($1, $2, $3, $4)", + args.point, args.action, uuid.UUID(args.attachment) if args.attachment else None, args.count) + print("armed %s -> %s x%d" % (args.point, args.action, args.count)) + await conn.close() + return 0 + + +async def cmd_republish(args) -> int: + conn = await connect() + job = await conn.fetchrow( + "SELECT * FROM jobs WHERE attachment_id = $1 AND ($2::int IS NULL OR pipeline_version = $2) " + "ORDER BY pipeline_version LIMIT 1", uuid.UUID(args.attachment_id), args.pipeline_version) + await conn.close() + if job is None: + print("no job", file=sys.stderr) + return 1 + attempt = args.attempt or max(1, job["attempt"]) + broker = Broker() + await broker.connect() + await broker.publish(job_payload(job["job_id"], job["attachment_id"], job["pipeline_version"], attempt)) + await broker.close() + print("published raw message job=%s v%d attempt=%d" % (job["job_id"], job["pipeline_version"], attempt)) + return 0 + + +async def cmd_dlq_peek(args) -> int: + broker = Broker() + await broker.connect() + q = await broker.channel.declare_queue(config.BROKER_DLQ, passive=True) + print(json.dumps({"dlq_messages": q.declaration_result.message_count})) + msg = await q.get(no_ack=False, fail=False) + if msg is not None: + print(json.dumps({"body": msg.body.decode("utf-8", "replace")[:500], + "x-death": str((msg.headers or {}).get("x-death"))[:300]}, ensure_ascii=False)) + await msg.nack(requeue=True) + await broker.close() + return 0 + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("failed"); s.add_argument("--limit", type=int, default=20); s.set_defaults(fn=cmd_failed) + s = sub.add_parser("replay"); s.add_argument("attachment_id"); s.add_argument("--actor", required=True); s.set_defaults(fn=cmd_replay) + s = sub.add_parser("reprocess"); s.add_argument("attachment_id"); s.add_argument("--pipeline-version", type=int, required=True) + s.add_argument("--actor", default="cli"); s.set_defaults(fn=cmd_reprocess) + s = sub.add_parser("inspect"); s.add_argument("attachment_id"); s.set_defaults(fn=cmd_inspect) + s = sub.add_parser("stats"); s.set_defaults(fn=cmd_stats) + s = sub.add_parser("fault"); fs = s.add_subparsers(dest="fault_cmd", required=True) + a = fs.add_parser("arm"); a.add_argument("point"); a.add_argument("action") + a.add_argument("--attachment"); a.add_argument("--count", type=int, default=1) + fs.add_parser("clear"); s.set_defaults(fn=cmd_fault) + s = sub.add_parser("republish"); s.add_argument("attachment_id"); s.add_argument("--attempt", type=int) + s.add_argument("--pipeline-version", type=int); s.set_defaults(fn=cmd_republish) + s = sub.add_parser("dlq-peek"); s.set_defaults(fn=cmd_dlq_peek) + args = p.parse_args() + return asyncio.run(args.fn(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/relay.py b/app/relay.py new file mode 100644 index 00000000..05357e5a --- /dev/null +++ b/app/relay.py @@ -0,0 +1,120 @@ +"""Transactional-outbox relay: PostgreSQL outbox -> RabbitMQ with publisher confirms. + + SELECT ... FOR UPDATE SKIP LOCKED -> publish + wait for confirm -> UPDATE published_at -> COMMIT + +A crash after the confirm but before COMMIT republishes the row later: a duplicate message, which the worker +treats as a no-op. A row is never marked published without a confirm, so a message accepted by the API is never +lost between COMMIT and publish. Several relays may run concurrently (SKIP LOCKED). +""" +from __future__ import annotations + +import asyncio +import json +import logging +import secrets +import signal +import time +import uuid + +import asyncpg + +from app import config, faults +from app.broker import Broker + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("relay") + +HEARTBEAT_FILE = "/tmp/relay-heartbeat" + + +class Relay: + def __init__(self): + self.id = "relay-" + secrets.token_hex(4) + self.pool = None + self.broker = Broker() + self.wakeup = asyncio.Event() + self.stopping = asyncio.Event() + + async def publish_batch(self) -> int: + async with self.pool.acquire() as conn, conn.transaction(): + rows = await conn.fetch( + """ + SELECT id, job_id, payload FROM outbox + WHERE published_at IS NULL AND available_at <= clock_timestamp() + ORDER BY available_at, id LIMIT $1 + FOR UPDATE SKIP LOCKED + """, + config.RELAY_BATCH, + ) + for row in rows: + payload = json.loads(row["payload"]) + await self.broker.publish(payload) # returns after the publisher confirm + await faults.hit(self.pool, "relay.after_confirm_before_mark", uuid.UUID(payload["attachment_id"]), self.id) + if rows: + await conn.execute( + """ + UPDATE outbox SET published_at = clock_timestamp(), publish_count = publish_count + 1 + WHERE id = ANY($1::bigint[]) + """, + [r["id"] for r in rows], + ) + return len(rows) + + async def listen(self) -> asyncpg.Connection: + conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT) + await conn.add_listener("outbox", lambda *_: self.wakeup.set()) + return conn + + async def run(self) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self.stopping.set) + listener = None + failures = 0 + while not self.stopping.is_set(): + try: + if self.pool is None: + self.pool = await asyncpg.create_pool(config.DATABASE_URL, min_size=1, max_size=3, + timeout=config.DB_CONNECT_TIMEOUT, + command_timeout=config.DB_COMMAND_TIMEOUT) + if self.broker.exchange is None: + await self.broker.connect() + logger.info("%s connected; fault_injection=%s", self.id, config.FAULT_INJECTION) + if listener is None or listener.is_closed(): + listener = await self.listen() + self.wakeup.clear() + published = await self.publish_batch() + failures = 0 + with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh: + fh.write(str(time.time())) + if published: + logger.info("published %d outbox message(s)", published) + if published == config.RELAY_BATCH: + continue + try: + # NOTIFY wakes us immediately; the poll interval covers delayed (backoff) rows. + await asyncio.wait_for(self.wakeup.wait(), config.RELAY_POLL_SECONDS) + except asyncio.TimeoutError: + pass + except Exception as exc: + failures += 1 + delay = min(10.0, 0.5 * 2 ** min(failures, 5)) + logger.warning("relay cycle failed (%s); retry in %.1fs; rows stay unpublished", type(exc).__name__, delay) + if self.broker.exchange is None or self.broker.connection.is_closed: + try: + await self.broker.close() + except Exception: + pass + self.broker = Broker() + if listener is not None and not listener.is_closed(): + try: + await listener.close() + except Exception: + pass + listener = None + await asyncio.sleep(delay) + await self.broker.close() + + +if __name__ == "__main__": + asyncio.run(Relay().run()) diff --git a/app/s3_bootstrap.py b/app/s3_bootstrap.py new file mode 100644 index 00000000..c81da8a1 --- /dev/null +++ b/app/s3_bootstrap.py @@ -0,0 +1,76 @@ +"""Idempotent bucket bootstrap with the admin identity (the API identity cannot create buckets). + +Checks afterwards that anonymous access to the bucket is denied, and warms up the write path: after a +SeaweedFS restart the first PUT waits ~2.5 s for the volume topology, which a client with a short timeout +would see as a failed upload. The API starts only after this one-shot service exits 0. +""" +from __future__ import annotations + +import io +import logging +import os +import sys +import time +import urllib.error +import urllib.request + +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.config import _secret +from app.objects import ObjectStore, S3Unavailable + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("s3-bootstrap") + + +def anonymous_status(path: str) -> int: + try: + with urllib.request.urlopen(config.S3_ENDPOINT.rstrip("/") + path, timeout=3) as r: + return r.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + admin = ObjectStore(_secret("S3_ADMIN_ACCESS_KEY_ID"), _secret("S3_ADMIN_SECRET_ACCESS_KEY")) + for attempt in range(1, 31): + try: + created = admin.create_bucket() + break + except (BotoCoreError, ClientError, S3Unavailable) as exc: + logger.warning("s3 not ready (attempt %d/30): %s", attempt, type(exc).__name__) + time.sleep(1) + else: + return 1 + logger.info("bucket %s %s", config.S3_BUCKET, "created" if created else "already exists") + + for path in ("/" + config.S3_BUCKET, "/" + config.S3_BUCKET + "/originals/probe"): + status = anonymous_status(path) + if status not in (401, 403): + logger.error("anonymous request %s returned %d: bucket is NOT private", path, status) + return 1 + logger.info("anonymous list/get denied: bucket is private") + + probe = "health/bootstrap-probe" # outside originals/, so the sweeper never looks at it + for attempt in range(1, 31): + started = time.monotonic() + try: + admin._s3.put_object(Bucket=admin.bucket, Key=probe, Body=io.BytesIO(b"probe")) + admin._s3.get_object(Bucket=admin.bucket, Key=probe)["Body"].read() + admin._s3.delete_object(Bucket=admin.bucket, Key=probe) + except (BotoCoreError, ClientError) as exc: + logger.warning("write probe failed (attempt %d/30): %s", attempt, type(exc).__name__) + time.sleep(1) + continue + elapsed = time.monotonic() - started + if elapsed < 0.5: + logger.info("write path warm: put/get/delete probe %.3fs", elapsed) + return 0 + logger.info("write probe slow (%.2fs), repeating", elapsed) + return 1 + + +if __name__ == "__main__": + os.umask(0o077) + sys.exit(main()) diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..6e834631 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,424 @@ +from __future__ import annotations + +import base64 +import json +import uuid +from datetime import datetime, timezone +from typing import List, Optional, Tuple + +import asyncpg + +from app import config, jobs +from app.errors import ApiError + + +def fmt_ts(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def user_row(row: asyncpg.Record) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": fmt_ts(row["created_at"]), + } + + +def chat_row(row: asyncpg.Record) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in row["member_ids"]], + "created_at": fmt_ts(row["created_at"]), + } + + +def message_row(row: asyncpg.Record, attachment_ids: Optional[List[uuid.UUID]] = None) -> dict: + if attachment_ids is None: + attachment_ids = row["attachment_ids"] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": fmt_ts(row["created_at"]), + "attachment_ids": [str(a) for a in attachment_ids], + } + + +def variant_row(row: asyncpg.Record) -> dict: + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + + +VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2} + + +def attachment_row(row: asyncpg.Record, variants=None) -> dict: + """Public view: variants of the currently published pipeline version, only when ready.""" + items = sorted((variant_row(v) for v in (variants or [])), key=lambda v: VARIANT_ORDER[v["name"]]) + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": items if row["status"] == "ready" else [], + "error_code": row["error_code"] if row["status"] == "failed" else None, + "created_at": fmt_ts(row["created_at"]), + "updated_at": fmt_ts(row["updated_at"]), + } + + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, object_key, " + "error_code, pipeline_version, created_at, updated_at") + + +def encode_cursor(resource: str, created_at: str, item_id: str) -> str: + payload = json.dumps({"r": resource, "t": created_at, "i": item_id}, separators=(",", ":")) + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(resource: str, cursor: str) -> Tuple[datetime, uuid.UUID]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")) + if payload.get("r") != resource: + raise ValueError("cursor belongs to a different resource") + created_at = datetime.fromisoformat(payload["t"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise ValueError("cursor time has no timezone") + return created_at, uuid.UUID(payload["i"]) + except Exception as exc: + raise ApiError(400, "invalid_request", "invalid cursor") from exc + + +def page_of(items: List[dict], resource: str, limit: int) -> dict: + page = items[:limit] + next_cursor = None + if len(items) > limit: + last = page[-1] + next_cursor = encode_cursor(resource, last["created_at"], last["id"]) + return {"items": page, "next_cursor": next_cursor} + + +class Store: + def __init__(self): + self.pool: Optional[asyncpg.Pool] = None + + async def open(self) -> None: + self.pool = await asyncpg.create_pool( + config.DATABASE_URL, + min_size=0, + max_size=config.DB_POOL_MAX, + timeout=config.DB_CONNECT_TIMEOUT, + command_timeout=config.DB_COMMAND_TIMEOUT, + max_inactive_connection_lifetime=60, + ) + + async def close(self) -> None: + if self.pool is not None: + await self.pool.close() + + def acquire(self): + return self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) + + async def ping(self) -> bool: + async with self.acquire() as conn: + return await conn.fetchval("SELECT 1") == 1 + + + async def create_user(self, username: str, display_name: str, password_hash: str) -> dict: + async with self.acquire() as conn: + try: + row = await conn.fetchrow( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at + """, + uuid.uuid4(), username, display_name, password_hash, + ) + except asyncpg.UniqueViolationError: + raise ApiError(409, "conflict", "username already taken") from None + return user_row(row) + + async def get_credentials(self, username: str) -> Optional[asyncpg.Record]: + async with self.acquire() as conn: + return await conn.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", + username, + ) + + + async def create_session(self, token_hash: bytes, user_id: uuid.UUID, ttl_seconds: int) -> datetime: + async with self.acquire() as conn: + return await conn.fetchval( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES ($1, $2, clock_timestamp() + make_interval(secs => $3)) + RETURNING expires_at + """, + token_hash, user_id, float(ttl_seconds), + ) + + async def find_session(self, token_hash: bytes) -> Optional[asyncpg.Record]: + async with self.acquire() as conn: + return await conn.fetchrow( + """ + SELECT s.expires_at, u.id, u.username, u.display_name, u.created_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp() + """, + token_hash, + ) + + async def revoke_session(self, token_hash: bytes) -> Optional[datetime]: + async with self.acquire() as conn: + return await conn.fetchval( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > clock_timestamp() + RETURNING expires_at + """, + token_hash, + ) + + + async def create_chat(self, title: str, member_ids: List[uuid.UUID]) -> dict: + chat_id = uuid.uuid4() + async with self.acquire() as conn: + try: + async with conn.transaction(): + row = await conn.fetchrow( + "INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at", + chat_id, title, + ) + await conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, m.user_id, m.ord - 1 + FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord) + """, + chat_id, member_ids, + ) + except asyncpg.ForeignKeyViolationError: + raise ApiError(400, "invalid_request", "unknown member in member_ids") from None + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in member_ids], + "created_at": fmt_ts(row["created_at"]), + } + + async def get_chat_for_member(self, chat_id: uuid.UUID, user_id: uuid.UUID) -> Optional[dict]: + async with self.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.position) AS member_ids + FROM chats c JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members me WHERE me.chat_id = c.id AND me.user_id = $2) + GROUP BY c.id + """, + chat_id, user_id, + ) + return chat_row(row) if row else None + + async def list_chats(self, user_id: uuid.UUID, limit: int, cursor: Optional[str]) -> dict: + after_t, after_i = decode_cursor("chats", cursor) if cursor else (None, None) + async with self.acquire() as conn: + rows = await conn.fetch( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.position) AS member_ids + FROM chat_members me + JOIN chats c ON c.id = me.chat_id + JOIN chat_members m ON m.chat_id = c.id + WHERE me.user_id = $1 + AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + GROUP BY c.id + ORDER BY c.created_at, c.id + LIMIT $4 + """, + user_id, after_t, after_i, limit + 1, + ) + return page_of([chat_row(r) for r in rows], "chats", limit) + + async def is_member(self, chat_id: uuid.UUID, user_id: uuid.UUID) -> bool: + async with self.acquire() as conn: + return bool(await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)", + chat_id, user_id, + )) + + + async def create_message( + self, chat_id: uuid.UUID, sender_id: uuid.UUID, text: str, attachment_ids: List[uuid.UUID], + key: str, request_hash: bytes, + ) -> Optional[dict]: + async with self.acquire() as conn: + for _ in range(3): + try: + async with conn.transaction(): + await conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + AND created_at < clock_timestamp() - make_interval(secs => $4) + """, + sender_id, chat_id, key, float(config.IDEMPOTENCY_TTL_SECONDS), + ) + message_id = uuid.uuid4() + claimed = await conn.fetchval( + """ + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) + SELECT $1, $2, $3, $4, $5 + WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1) + ON CONFLICT (user_id, chat_id, key) DO NOTHING + RETURNING message_id + """, + sender_id, chat_id, key, request_hash, message_id, + ) + if claimed is not None: + if attachment_ids: + # lab5: queued/processing/ready may be linked; failed -> 409; unknown/foreign -> 404 + found = await conn.fetch( + "SELECT status FROM attachments WHERE id = ANY($1::uuid[]) AND chat_id = $2 FOR SHARE", + attachment_ids, chat_id, + ) + if len(found) != len(attachment_ids): + raise ApiError(404, "not_found", "attachment not found") + if any(r["status"] == "failed" for r in found): + raise ApiError(409, "conflict", "failed attachment cannot be attached") + row = await conn.fetchrow( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at + """, + message_id, chat_id, sender_id, text, + ) + if attachment_ids: + await conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) + SELECT $1, a.id, $2, a.ord - 1 + FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord) + """, + message_id, chat_id, attachment_ids, + ) + return message_row(row, attachment_ids) + except asyncpg.ForeignKeyViolationError: + return None + + existing = await conn.fetchrow( + """ + SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM idempotency_keys k + JOIN messages m ON m.id = k.message_id + JOIN chat_members cm ON cm.chat_id = k.chat_id AND cm.user_id = k.user_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3 + """, + sender_id, chat_id, key, + ) + if existing is None: + is_member = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)", + chat_id, sender_id, + ) + if not is_member: + return None + continue + if existing["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key was already used with a different request") + return message_row(existing) + raise ApiError(503, "unavailable", "could not settle idempotent request, retry with the same key") + + async def list_messages(self, chat_id: uuid.UUID, limit: int, cursor: Optional[str]) -> dict: + resource = str(chat_id) + async with self.acquire() as conn: + if cursor: + after_t, after_i = decode_cursor(resource, cursor) + rows = await conn.fetch( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM messages m + WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3) + ORDER BY m.created_at, m.id + LIMIT $4 + """, + chat_id, after_t, after_i, limit + 1, + ) + else: + rows = await conn.fetch( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM messages m + WHERE m.chat_id = $1 + ORDER BY m.created_at, m.id + LIMIT $2 + """, + chat_id, limit + 1, + ) + return page_of([message_row(r) for r in rows], resource, limit) + + + async def create_queued_attachment( + self, attachment_id: uuid.UUID, chat_id: uuid.UUID, owner_id: uuid.UUID, filename: str, + content_type: str, size_bytes: int, sha256: str, width: int, height: int, + staging_key: str, object_key: str, actor: str, + ) -> Optional[dict]: + """Durable handoff: attachment(queued) + job + outbox row in ONE transaction, after the staging PUT.""" + async with self.acquire() as conn: + try: + async with conn.transaction(): + row = await conn.fetchrow( + f""" + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, staging_key, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued') + RETURNING {ATTACHMENT_COLUMNS} + """, + attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, + object_key, staging_key, + ) + await jobs.create_job(conn, attachment_id, config.PIPELINE_VERSION, actor) + except asyncpg.ForeignKeyViolationError: + return None + return attachment_row(row) + + async def get_attachment_for_member(self, attachment_id: uuid.UUID, user_id: uuid.UUID): + async with self.acquire() as conn: + row = await conn.fetchrow( + f""" + SELECT {ATTACHMENT_COLUMNS} FROM attachments a + WHERE a.id = $1 + AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2) + """, + attachment_id, user_id, + ) + if row is None or row["status"] != "ready": + return row, [] + variants = await conn.fetch( + """ + SELECT name, object_key, content_type, size_bytes, width, height, sha256 + FROM attachment_variants WHERE attachment_id = $1 AND pipeline_version = $2 + """, + attachment_id, row["pipeline_version"], + ) + return row, variants diff --git a/app/sweeper.py b/app/sweeper.py new file mode 100644 index 00000000..52e318e4 --- /dev/null +++ b/app/sweeper.py @@ -0,0 +1,162 @@ +"""Staging/orphan cleanup with documented retention. Never touches work that is still in progress. + +Rules (every step idempotent, repeated runs are safe): + 1. staging/ of a READY attachment -> delete (the worker deletes it right after COMMIT; + this repairs a crash/failed delete in between) + 2. staging/ of a FAILED attachment older than STAGING_RETENTION_SECONDS -> delete, mark staging_deleted_at + (until then an operator replay can still use it; afterwards replay ends as failed staging_missing) + 3. staging/ without any attachment row, older than ORPHAN_GRACE_SECONDS -> delete + (API crashed/failed between the staging PUT and the COMMIT of attachment+job+outbox; no 202 was sent) + 4. attachments//... without any attachment row, or not referenced by any committed row (superseded + older pipeline version, fenced-out attempt) while no job of that attachment is active, older than grace + 5. published outbox rows older than 7 days -> delete +Queued/processing attachments are never touched: their staging object is the only copy of accepted work. + + python -m app.sweeper --once --dry-run # report only + python -m app.sweeper --once # one real pass + python -m app.sweeper # loop every SWEEP_INTERVAL_SECONDS +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import time +import uuid +from datetime import datetime, timedelta, timezone + +import asyncpg +from starlette.concurrency import run_in_threadpool + +from app import config +from app.objects import FINAL_PREFIX, STAGING_PREFIX, ObjectStore, S3Unavailable + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("sweeper") + +LOCK_ID = 2026_0005 +HEARTBEAT_FILE = "/tmp/sweeper-heartbeat" + + +def _id_from_key(key: str, prefix: str): + part = key[len(prefix):].split("/", 1)[0] + try: + return uuid.UUID(hex=part) + except ValueError: + return None + + +async def sweep(conn: asyncpg.Connection, objects: ObjectStore, grace: int, retention: int, dry_run: bool) -> dict: + started = time.monotonic() + report = {"dry_run": dry_run, "grace_seconds": grace, "retention_seconds": retention, + "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_orphans": 0, "final_orphans": 0, + "outbox_pruned": 0, "objects_scanned": 0} + now = datetime.now(timezone.utc) + cutoff = now - timedelta(seconds=grace) + + # 1 + 2: staging of finished attachments + rows = await conn.fetch( + """ + SELECT id, staging_key, status FROM attachments + WHERE staging_deleted_at IS NULL AND staging_key IS NOT NULL + AND (status = 'ready' + OR (status = 'failed' AND updated_at < clock_timestamp() - make_interval(secs => $1))) + """, float(retention)) + for row in rows: + report["staging_ready_deleted" if row["status"] == "ready" else "staging_failed_expired"] += 1 + if dry_run: + continue + await run_in_threadpool(objects.delete_sync, row["staging_key"]) + await conn.execute( + "UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = $1 AND status IN ('ready', 'failed')", + row["id"]) + + # 3 + 4: objects without any row (no 202 was ever sent for them) + for prefix, counter in ((STAGING_PREFIX, "staging_orphans"), (FINAL_PREFIX, "final_orphans")): + def old_keys(): + out = [] + for key, modified in objects.list_prefix(prefix): + report["objects_scanned"] += 1 + if modified < cutoff: + out.append(key) + return out + keys = await run_in_threadpool(old_keys) + by_id = {} + for key in keys: + ident = _id_from_key(key, prefix) + if ident is not None: + by_id.setdefault(ident, []).append(key) + if not by_id: + continue + known = {r["id"] for r in await conn.fetch("SELECT id FROM attachments WHERE id = ANY($1::uuid[])", list(by_id))} + orphans = [k for ident, ks in by_id.items() if ident not in known for k in ks] + if prefix == FINAL_PREFIX: + # objects of a known attachment that no committed row references: a superseded older pipeline + # version or a fenced-out attempt. Only after the grace period (longer than any live attempt). + candidates = [k for ident, ks in by_id.items() if ident in known for k in ks] + if candidates: + referenced = {r["k"] for r in await conn.fetch( + """ + SELECT object_key AS k FROM attachments WHERE object_key = ANY($1::text[]) AND status = 'ready' + UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY($1::text[]) + UNION SELECT a.object_key FROM attachments a WHERE a.object_key = ANY($1::text[]) + AND a.status IN ('queued', 'processing') + """, candidates)} + live = {r["id"] for r in await conn.fetch( + "SELECT DISTINCT attachment_id AS id FROM jobs WHERE status IN ('queued', 'processing')")} + orphans += [k for k in candidates if k not in referenced and _id_from_key(k, prefix) not in live] + report[counter] += len(orphans) + if orphans and not dry_run: + await run_in_threadpool(objects.delete_many, orphans) + + # 5: outbox history + if not dry_run: + res = await conn.execute( + "DELETE FROM outbox WHERE published_at < clock_timestamp() - interval '7 days'") + report["outbox_pruned"] = int(res.split()[-1]) + + report["duration_ms"] = round((time.monotonic() - started) * 1000, 1) + return report + + +async def run_once(grace: int, retention: int, dry_run: bool) -> dict: + conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT) + try: + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_ID): + return {"skipped": "another sweeper holds the lock"} + try: + return await sweep(conn, ObjectStore(), grace, retention, dry_run) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID) + finally: + await conn.close() + + +async def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--once", action="store_true") + p.add_argument("--dry-run", action="store_true") + p.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS) + p.add_argument("--retention", type=int, default=config.STAGING_RETENTION_SECONDS) + p.add_argument("--interval", type=int, default=config.SWEEP_INTERVAL_SECONDS) + args = p.parse_args() + if args.grace < 60: + p.error("grace below 60 s could remove an upload that is still between staging PUT and COMMIT") + while True: + try: + report = await run_once(args.grace, args.retention, args.dry_run) + print(json.dumps(report, ensure_ascii=False), flush=True) + with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh: + fh.write(str(time.time())) + except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError, S3Unavailable) as exc: + logger.warning("sweep failed (%s); will retry", type(exc).__name__) + if args.once: + return 1 + if args.once: + return 0 + await asyncio.sleep(args.interval) + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..fcb7a967 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,377 @@ +"""Attachment worker: consumes attachments.process, publishes the original and variants, commits the result. + +Delivery is at-least-once; the effect is idempotent because + * PostgreSQL (jobs) decides whether a delivery may start an attempt: claim = queued -> processing with + attempt+1 and a lease (lease_owner, lease_until). Duplicates/stale/reordered messages are acked as no-ops. + * (job_id, attempt) is a fencing token: the result COMMIT succeeds only if this worker still owns the lease. + * S3 keys are the effect key (attachment, pipeline_version, variant): a retry overwrites identical bytes. + * attachment_variants has PRIMARY KEY (attachment_id, pipeline_version, name): one row per variant. + * A result of an older pipeline_version never replaces a newer published one (superseded). +The message is acked only after the result (or the decision) is durably committed in PostgreSQL. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import secrets +import signal +import tempfile +import time +import uuid +from typing import Optional + +import asyncpg +from aio_pika.abc import AbstractIncomingMessage + +from app import config, faults, jobs +from app.broker import Broker, InvalidJob, parse_job +from app.imaging import ImageRejected, TransientFailure, run_isolated +from app.objects import ObjectStore, S3NotFound, S3Unavailable, original_key, variant_key + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("worker") + +HEARTBEAT_FILE = "/tmp/worker-heartbeat" +DB_ERRORS = (OSError, asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.TooManyConnectionsError, + asyncpg.exceptions.OperatorInterventionError, asyncio.TimeoutError) + + +class Worker: + def __init__(self): + self.id = "worker-" + secrets.token_hex(4) + self.pool: Optional[asyncpg.Pool] = None + self.objects = ObjectStore() + self.broker = Broker() + self.inflight = set() + self.stopping = asyncio.Event() + + # ------------------------------------------------------------------ message entry point + async def on_message(self, message: AbstractIncomingMessage) -> None: + task = asyncio.current_task() + self.inflight.add(task) + try: + await self._handle(message) + finally: + self.inflight.discard(task) + + async def _handle(self, message: AbstractIncomingMessage) -> None: + try: + job = parse_job(message.body) + except InvalidJob as exc: + logger.error("rejecting malformed job message to DLQ: %s", exc) + await message.reject(requeue=False) # -> attachments.dlx -> attachments.dlq + return + try: + outcome = await self.process(job, message.redelivered) + except DB_ERRORS as exc: + # Nothing was decided durably. Give the message back after a pause (no hot loop); if the DB stays down + # the lease expires and the reaper re-enqueues the job through the outbox. + logger.warning("db unavailable while handling job %s: %s; requeue", job["job_id"], type(exc).__name__) + await asyncio.sleep(2) + await message.nack(requeue=True) + return + await faults.hit(self.pool, "worker.after_commit_before_ack", job["attachment_id"], self.id) + await message.ack() + logger.info("job %s attempt %s -> %s (redelivered=%s)", job["job_id"], job["attempt"], outcome, message.redelivered) + + # ------------------------------------------------------------------ claim + async def claim(self, job: dict, redelivered: bool): + async with self.pool.acquire() as conn, conn.transaction(): + row = await conn.fetchrow( + """ + SELECT j.*, a.status AS a_status, a.content_type, a.size_bytes, a.sha256, a.staging_key, + a.staging_deleted_at, a.object_key, a.pipeline_version AS a_version, + (j.lease_until > clock_timestamp()) AS lease_active, + (j.next_attempt_at <= clock_timestamp() + interval '1 second') AS due + FROM jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.job_id = $1 + FOR UPDATE OF j + """, + job["job_id"], + ) + reason = None + if row is None: + reason = "unknown_job_noop" + elif row["attachment_id"] != job["attachment_id"] or row["pipeline_version"] != job["pipeline_version"]: + reason = "mismatched_message_noop" + elif row["status"] == "done": + reason = "duplicate_noop" # result already committed: ack after checking state + elif row["status"] in ("failed", "superseded"): + reason = row["status"] + "_noop" + elif row["status"] == "processing": + reason = "lease_held_noop" if row["lease_active"] else "lease_expired_noop" + elif job["attempt"] != row["attempt"] + 1: + reason = "stale_attempt_noop" # reordered/old message of an earlier attempt + elif not row["due"]: + reason = "early_noop" + elif row["a_version"] is not None and row["a_version"] > row["pipeline_version"]: + await conn.execute( + "UPDATE jobs SET status = 'superseded', updated_at = clock_timestamp() WHERE job_id = $1", row["job_id"]) + reason = "superseded_before_start" + if reason: + await jobs.event(conn, job["job_id"], job["attachment_id"] if row is None else row["attachment_id"], + reason, self.id, job["attempt"], "redelivered=%s" % redelivered) + return None, reason + claimed = await conn.fetchrow( + """ + UPDATE jobs SET status = 'processing', attempt = attempt + 1, lease_owner = $2, + lease_until = clock_timestamp() + make_interval(secs => $3), updated_at = clock_timestamp() + WHERE job_id = $1 + RETURNING * + """, + row["job_id"], self.id, config.LEASE_SECONDS, + ) + await conn.execute( + "UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = $1 AND status = 'queued'", + row["attachment_id"], + ) + await jobs.event(conn, row["job_id"], row["attachment_id"], "claimed", self.id, claimed["attempt"], + "redelivered=%s" % redelivered) + info = dict(row) + info.update(dict(claimed)) + return info, "claimed" + + async def heartbeat(self, job: dict, lost: asyncio.Event) -> None: + """Renew the lease while the attempt is alive; a dead worker stops renewing and the lease expires.""" + while True: + await asyncio.sleep(config.HEARTBEAT_SECONDS) + try: + async with self.pool.acquire() as conn: + renewed = await conn.fetchval( + """ + UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => $4) + WHERE job_id = $1 AND status = 'processing' AND lease_owner = $2 AND attempt = $3 + RETURNING 1 + """, + job["job_id"], self.id, job["attempt"], config.LEASE_SECONDS, + ) + except DB_ERRORS: + continue + if not renewed: + lost.set() + return + + # ------------------------------------------------------------------ process one delivery + async def process(self, job: dict, redelivered: bool) -> str: + claimed, reason = await self.claim(job, redelivered) + if claimed is None: + return reason + await faults.hit(self.pool, "worker.after_claim", claimed["attachment_id"], self.id) + lost = asyncio.Event() + hb = asyncio.create_task(self.heartbeat(claimed, lost)) + try: + with tempfile.TemporaryDirectory(prefix="job-") as tmp: + try: + variants = await self.execute(claimed, tmp) + await faults.hit(self.pool, "worker.after_s3_before_commit", claimed["attachment_id"], self.id) + except ImageRejected as exc: + return await self.finish_failure(claimed, exc.code, permanent=True) + except (TransientFailure, S3Unavailable) as exc: + code = exc.code if isinstance(exc, TransientFailure) else "s3_unavailable" + return await self.finish_failure(claimed, code, permanent=False) + return await self.commit(claimed, variants) + finally: + hb.cancel() + + async def execute(self, job: dict, tmp: str) -> list: + """Side effects only in S3 at deterministic keys; nothing visible to clients until commit().""" + src = os.path.join(tmp, "source") + republish = job["a_status"] == "ready" # newer pipeline for an already published attachment + if not republish and job["staging_deleted_at"] is not None: + raise ImageRejected("staging_missing") + # Source: staging, or the already published original (newer pipeline / a concurrent version committed + # and removed staging in between). Both hold the same bytes, verified against the DB checksum below. + keys = [job["object_key"]] if republish else [job["staging_key"], job["object_key"]] + size = None + for key in keys: + try: + with open(src, "wb") as fh: + size = await asyncio.to_thread(self.objects.download_sync, key, fh, config.MAX_UPLOAD_BYTES) + break + except S3NotFound: + continue + if size is None: + raise ImageRejected("staging_missing") + digest = hashlib.sha256() + with open(src, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]: + raise ImageRejected("staging_corrupt") # trusted metadata from the DB disagrees with the bytes + await faults.hit(self.pool, "worker.before_transform", job["attachment_id"], self.id) + out = os.path.join(tmp, "out") + os.mkdir(out) + started = time.monotonic() + variants = await asyncio.to_thread(run_isolated, src, job["content_type"], out, job["pipeline_version"], + config.JOB_TIMEOUT_SECONDS, config.WORKER_MEMORY_LIMIT_MB) + logger.info("transformed %s v%d in %.0f ms", job["attachment_id"], job["pipeline_version"], + 1000 * (time.monotonic() - started)) + if not republish: + with open(src, "rb") as fh: + await asyncio.to_thread(self.objects.put_sync, original_key(job["attachment_id"]), fh, size, + job["content_type"], job["sha256"]) + for v in variants: + v["object_key"] = variant_key(job["attachment_id"], job["pipeline_version"], v["name"], v["content_type"]) + with open(v["path"], "rb") as fh: + await asyncio.to_thread(self.objects.put_sync, v["object_key"], fh, v["size_bytes"], + v["content_type"], v["sha256"]) + return variants + + async def fenced(self, conn: asyncpg.Connection, job: dict): + row = await conn.fetchrow("SELECT * FROM jobs WHERE job_id = $1 FOR UPDATE", job["job_id"]) + if row is None or row["status"] != "processing" or row["lease_owner"] != self.id or row["attempt"] != job["attempt"]: + await jobs.event(conn, job["job_id"], job["attachment_id"], "fenced_out", self.id, job["attempt"], + "lease lost; result of this attempt discarded") + return None + return row + + async def commit(self, job: dict, variants: list) -> str: + async with self.pool.acquire() as conn: + async with conn.transaction(): + row = await self.fenced(conn, job) + if row is None: + return "fenced_out" + current = await conn.fetchval("SELECT pipeline_version FROM attachments WHERE id = $1 FOR UPDATE", + job["attachment_id"]) + if current is not None and current > job["pipeline_version"]: + await conn.execute( + """ + UPDATE jobs SET status = 'superseded', lease_owner = NULL, lease_until = NULL, + updated_at = clock_timestamp() WHERE job_id = $1 + """, job["job_id"]) + await jobs.event(conn, job["job_id"], job["attachment_id"], "superseded", self.id, job["attempt"], + "published v%d is newer than v%d" % (current, job["pipeline_version"])) + return "superseded" + for v in variants: + await conn.execute( + """ + INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, + size_bytes, width, height, sha256, job_id, attempt) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"], job["job_id"], job["attempt"], + ) + await conn.execute( + """ + UPDATE attachments SET status = 'ready', pipeline_version = $2, error_code = NULL, + updated_at = clock_timestamp() + WHERE id = $1 + """, + job["attachment_id"], job["pipeline_version"], + ) + await conn.execute( + """ + UPDATE jobs SET status = 'done', lease_owner = NULL, lease_until = NULL, error_code = NULL, + last_error = NULL, updated_at = clock_timestamp() + WHERE job_id = $1 + """, job["job_id"]) + await jobs.event(conn, job["job_id"], job["attachment_id"], "ready", self.id, job["attempt"], + "v%d: %s" % (job["pipeline_version"], ",".join(v["name"] for v in variants))) + # Staging is removed only after the result is committed; a failure here is repaired by the sweeper. + if job["a_status"] != "ready" and job["staging_deleted_at"] is None: + try: + await asyncio.to_thread(self.objects.delete_sync, job["staging_key"]) + await conn.execute( + "UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = $1 AND status = 'ready'", + job["attachment_id"]) + except (S3Unavailable, *DB_ERRORS): + logger.warning("staging cleanup deferred to sweeper for %s", job["attachment_id"]) + return "ready" + + async def finish_failure(self, job: dict, code: str, permanent: bool) -> str: + async with self.pool.acquire() as conn, conn.transaction(): + row = await self.fenced(conn, job) + if row is None: + return "fenced_out" + if permanent: + await jobs.fail(conn, row, code, code, self.id) + return "failed" + return await jobs.retry_or_fail(conn, row, code, self.id) + + # ------------------------------------------------------------------ reaper: expired leases, lost messages + async def reap_once(self) -> None: + async with self.pool.acquire() as conn: + async with conn.transaction(): + expired = await conn.fetch( + """ + SELECT * FROM jobs WHERE status = 'processing' AND lease_until < clock_timestamp() + ORDER BY lease_until LIMIT 50 FOR UPDATE SKIP LOCKED + """) + for row in expired: + await jobs.event(conn, row["job_id"], row["attachment_id"], "lease_expired", self.id, row["attempt"], + "owner %s stopped renewing" % row["lease_owner"]) + await jobs.retry_or_fail(conn, row, "lease_expired", self.id) + async with conn.transaction(): + # Safety net for a message lost after the outbox marked it published (e.g. broker volume lost). + stuck = await conn.fetch( + """ + SELECT j.* FROM jobs j + WHERE j.status = 'queued' AND j.next_attempt_at < clock_timestamp() - make_interval(secs => $1) + AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id + AND (o.published_at IS NULL OR o.published_at > clock_timestamp() - make_interval(secs => $1))) + LIMIT 50 FOR UPDATE OF j SKIP LOCKED + """, config.STUCK_QUEUED_SECONDS) + for row in stuck: + await jobs.enqueue(conn, row["job_id"], row["attachment_id"], row["pipeline_version"], row["attempt"] + 1) + await jobs.event(conn, row["job_id"], row["attachment_id"], "republished_stuck", self.id, row["attempt"]) + with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh: + fh.write(str(time.time())) + + async def reaper(self) -> None: + while not self.stopping.is_set(): + try: + await self.reap_once() + except DB_ERRORS as exc: + logger.warning("reaper: db unavailable (%s)", type(exc).__name__) + except Exception: + logger.exception("reaper failed") + try: + await asyncio.wait_for(self.stopping.wait(), config.REAPER_INTERVAL_SECONDS) + except asyncio.TimeoutError: + pass + + # ------------------------------------------------------------------ lifecycle + async def run(self) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self.stopping.set) + for attempt in range(1, 61): + try: + self.pool = await asyncpg.create_pool(config.DATABASE_URL, min_size=1, + max_size=config.WORKER_CONCURRENCY + 4, + timeout=config.DB_CONNECT_TIMEOUT, + command_timeout=config.DB_COMMAND_TIMEOUT) + await self.broker.connect(prefetch=config.WORKER_CONCURRENCY) + break + except Exception as exc: + logger.warning("dependencies not ready (attempt %d): %s", attempt, type(exc).__name__) + await asyncio.sleep(1) + else: + raise SystemExit("cannot connect to PostgreSQL/RabbitMQ") + logger.info("%s consuming %s, prefetch=%d, pipeline=%d, max_attempts=%d, lease=%.0fs, fault_injection=%s", + self.id, config.BROKER_QUEUE, config.WORKER_CONCURRENCY, config.PIPELINE_VERSION, + config.JOB_MAX_ATTEMPTS, config.LEASE_SECONDS, config.FAULT_INJECTION) + reaper = asyncio.create_task(self.reaper()) + tag = await self.broker.queue.consume(self.on_message, no_ack=False) + await self.stopping.wait() + logger.info("stopping: no new deliveries, waiting for %d in-flight jobs", len(self.inflight)) + await self.broker.queue.cancel(tag) + if self.inflight: + await asyncio.wait(list(self.inflight), timeout=25) + reaper.cancel() + await self.broker.close() # unacked deliveries (if any) return to the queue + await self.pool.close() + + +def main() -> None: + if not config.DATABASE_URL: + raise SystemExit("DATABASE_URL(_FILE) is required") + asyncio.run(Worker().run()) + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..ab14eaaa --- /dev/null +++ b/compose.yaml @@ -0,0 +1,360 @@ +name: messenger-lab5 + +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +x-pipeline-env: &pipeline-env + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + FAULT_INJECTION: ${FAULT_INJECTION:-0} + +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + stop_grace_period: 30s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1.4-alpine3.22 + command: ["valkey-server", "/run/secrets/valkey_conf"] + secrets: [valkey_conf] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "valkey-cli ping 2>&1 | grep -qE 'PONG|NOAUTH'"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "0.25", memory: 96M} + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -s3 + - -s3.ip.bind=0.0.0.0 + - -s3.port=9000 + - -s3.config=/run/secrets/seaweedfs_s3_config + - -master.volumeSizeLimitMB=1024 + - -volume.max=0 + - -s3.concurrentUploadLimitMB=128 + - -filer.concurrentUploadLimitMB=128 + - -volume.concurrentUploadLimitMB=128 + environment: + GOMEMLIMIT: 450MiB + secrets: [seaweedfs_s3_config] + volumes: + - s3data:/data + networks: [data] + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/healthz && wget -q -O - http://127.0.0.1:9333/dir/status | grep -q '\"DataNodes\":\\[{'"] + interval: 2s + timeout: 3s + start_period: 5s + retries: 30 + deploy: + resources: + limits: {cpus: "1.0", memory: 768M} + stop_grace_period: 20s + restart: unless-stopped + + # Broker: quorum queues (Raft log on disk) + publisher confirms + manual consumer acks. + rabbitmq: + image: rabbitmq:4.1.4-management-alpine + hostname: rabbitmq # node name rabbit@rabbitmq must be stable for the data volume + secrets: + - source: rabbitmq_conf + target: /etc/rabbitmq/conf.d/20-messenger.conf + volumes: + - rabbitmqdata:/var/lib/rabbitmq + networks: [data, mgmt] + ports: + # management UI only on loopback, for the operator during the defence + - "127.0.0.1:${RABBITMQ_UI_PORT:-15672}:15672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 5s + timeout: 10s + start_period: 20s + retries: 30 + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + stop_grace_period: 30s + restart: unless-stopped + + s3-init: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.s3_bootstrap"] + environment: + <<: *s3-env + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: [s3_admin_access_key_id, s3_admin_secret_access_key] + networks: [data] + depends_on: + s3: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: "no" + + migrate: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: "no" + + api: + build: . + image: mtusi-messenger:lab5 + environment: + <<: [*s3-env, *pipeline-env] + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: postgres + SESSION_CACHE_MAX_SECONDS: ${SESSION_CACHE_MAX_SECONDS:-300} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MAX: ${DB_POOL_MAX:-10} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-16} + DECODE_CONCURRENCY: ${DECODE_CONCURRENCY:-2} + UPLOAD_SPOOL_BYTES: ${UPLOAD_SPOOL_BYTES:-1048576} + S3_PUT_CONCURRENCY: ${S3_PUT_CONCURRENCY:-4} + # the API never talks to the broker: it writes the outbox; broker_url is not mounted here + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data] + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"] + interval: 5s + timeout: 3s + start_period: 5s + retries: 3 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "0.5", memory: 384M} + stop_signal: SIGTERM + stop_grace_period: 10s + restart: unless-stopped + + # Outbox -> RabbitMQ (publisher confirms). Separate process: its crash/stop never blocks 202 uploads. + relay: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.relay"] + environment: + <<: *pipeline-env + RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-1} + secrets: [database_url, broker_url] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/relay-heartbeat') < 30 else 1)"] + interval: 10s + timeout: 3s + start_period: 15s + retries: 3 + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + stop_grace_period: 10s + restart: unless-stopped + + worker: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.worker"] + environment: + <<: [*s3-env, *pipeline-env] + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + LEASE_SECONDS: ${LEASE_SECONDS:-20} + HEARTBEAT_SECONDS: ${HEARTBEAT_SECONDS:-5} + RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2} + RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60} + WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-768} + STUCK_QUEUED_SECONDS: ${STUCK_QUEUED_SECONDS:-300} + S3_READ_TIMEOUT: "15" + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 30 else 1)"] + interval: 10s + timeout: 3s + start_period: 20s + retries: 3 + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: {cpus: "${WORKER_CPUS:-1.0}", memory: "${WORKER_MEMORY:-1024M}"} + stop_signal: SIGTERM + stop_grace_period: 30s + restart: unless-stopped + + # Operator toolbox: `docker compose exec ops python -m app.ops ...` (replay, failed registry, inspect, faults). + ops: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-c", "import signal; signal.signal(signal.SIGTERM, lambda *a: exit(0)); signal.pause()"] + healthcheck: + test: ["CMD", "true"] + interval: 30s + start_period: 1s + environment: + <<: [*s3-env, *pipeline-env] + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: unless-stopped + + sweeper: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.sweeper"] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/sweeper-heartbeat') < 3*int(os.environ.get('SWEEP_INTERVAL_SECONDS','60'))+30 else 1)"] + interval: 10s + timeout: 3s + start_period: 10s + retries: 3 + environment: + <<: *s3-env + DATABASE_URL_FILE: /run/secrets/database_url + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-900} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-60} + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: unless-stopped + + lb: + image: haproxy:3.2.4-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + networks: [edge] + depends_on: + api: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8405/lb-health"] + interval: 5s + timeout: 2s + retries: 3 + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: unless-stopped + +networks: + edge: {} + mgmt: {} # only for the loopback-published RabbitMQ management UI + data: + internal: true + +volumes: + pgdata: + s3data: + rabbitmqdata: + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + valkey_conf: + file: ./secrets/valkey.conf + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + seaweedfs_s3_config: + file: ./secrets/seaweedfs_s3.json + rabbitmq_conf: + file: ./secrets/rabbitmq.conf + broker_url: + file: ./secrets/broker_url diff --git a/evidence/README.md b/evidence/README.md index 8df70015..851bb066 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,18 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Файлы этой работы (lab5) + +| Файл | Что доказывает | Команда | +| --- | --- | --- | +| `smoke.txt` | публичный сценарий lab5 | `make test` | +| `pipeline.txt` | варианты (256×160, crop 128, WebP), EXIF, alpha, лимиты 413/415, invalid_image без повторов | `make test-pipeline` | +| `faults.txt`, `faults.json` | 14 сценариев отказов (уровни 3–4, трек 2) с инвариантами I1–I5 | `make test-faults` (FAULT_INJECTION=1) | +| `limits.txt` | лимиты памяти (RLIMIT_AS) и времени worker | `make test-limits` | +| `persistence.txt` | рестарт API, worker, relay и всех хранилищ с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | +| `failover.txt`, `stop-one.txt` | отказ API-реплики (регресс lab3) | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | +| `sweeper.txt` | retention/очистка неиспользуемых объектов, идемпотентность sweeper | см. заголовок файла | +| `load.json` | регресс синхронного пути сообщений | `tests/load.py` (см. REPORT.md) | + +Стенд: Apple M3, 8 CPU, 16 GiB, Docker Desktop 28.0.1. В файлах нет токенов и паролей: тесты держат токены только в памяти, hook-вывод скрыт. diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..b163b39e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,24 @@ +# failover: stop the API replica that served the test, twice (restore in between) +# date: 2026-11-10T13:49:42Z + +## run 1: make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 0, "first_recovery_seconds": 3.483} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +NAME STATUS +messenger-lab5-api-1 Up 5 seconds (healthy) +messenger-lab5-api-2 Up 39 seconds (healthy) + +## run 2: make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 0, "first_recovery_seconds": 2.809} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +NAME STATUS +messenger-lab5-api-1 Up 5 seconds (healthy) +messenger-lab5-api-2 Up About a minute (healthy) + +## stop-one.sh log +2026-11-10T13:49:45Z stopped messenger-lab5-api-1 (45db2233b0c2) instance=api-cf4d97a2 state=exited +2026-11-10T13:50:09Z stopped messenger-lab5-api-1 (45db2233b0c2) instance=api-3423c899 state=exited diff --git a/evidence/faults.json b/evidence/faults.json new file mode 100644 index 00000000..b2b86c1d --- /dev/null +++ b/evidence/faults.json @@ -0,0 +1,133 @@ +{ + "scenarios": [ + { + "scenario": "worker_down", + "what": "worker stopped -> upload 202 -> message -> API + broker restart -> worker start -> ready", + "seconds": 31.1, + "observed": [ + "worker: messenger-lab5-worker-1 exited Exited (0) Less than a second ago", + "queue before broker restart: {\"jobs\": {\"superseded\": 1, \"failed\": 6, \"done\": 33, \"queued\": 1}, \"outbox_unpublished\": 0, \"queue_ready_messages\": 1, \"queue_consumers\": 0, \"dlq_messages\": 1}", + "queue after broker restart (volume kept): {\"jobs\": {\"superseded\": 1, \"failed\": 6, \"done\": 33, \"queued\": 1}, \"outbox_unpublished\": 0, \"queue_ready_messages\": 1, \"queue_consumers\": 0, \"dlq_messages\": 1}", + "attachment 1f804983-862e-4fcc-b3ca-ee2af8564aba ready after worker start; thumbnail 256x160" + ] + }, + { + "scenario": "outbox_gap", + "what": "COMMIT done, relay down (= crash between COMMIT and publish) -> relay restart publishes -> ready", + "seconds": 15.8, + "observed": [ + "relay down: outbox [{'id': 60, 'attempt': '1', 'published': False, 'publish_count': 0}], attachment queued", + "after relay start: ready, events ['accepted', 'claimed', 'ready']" + ] + }, + { + "scenario": "relay_crash_after_confirm", + "what": "relay dies after the publisher confirm but before marking the row -> duplicate publish -> worker no-op", + "seconds": 7.6, + "observed": [ + "events: ['accepted', 'fault_injected', 'claimed', 'ready', 'duplicate_noop']" + ] + }, + { + "scenario": "crash_after_claim", + "what": "crash before any side effect: lease expires, reaper re-enqueues attempt 2", + "seconds": 28.5, + "observed": [ + "worker.after_claim: ready in 23.9s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2]" + ] + }, + { + "scenario": "crash_after_s3", + "what": "crash after writing S3 objects, before COMMIT: retry overwrites the same keys, one set of variants", + "seconds": 29.0, + "observed": [ + "worker.after_s3_before_commit: ready in 24.2s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2]" + ] + }, + { + "scenario": "crash_after_commit", + "what": "crash after COMMIT, before ack: redelivery sees done -> ack as no-op, no second attempt", + "seconds": 6.6, + "observed": [ + "worker.after_commit_before_ack: ready in 1.7s; events ['accepted', 'claimed', 'ready', 'fault_injected', 'duplicate_noop']; variant attempts [1]" + ] + }, + { + "scenario": "transient_backoff", + "what": "two transient failures -> retries with exponential backoff + jitter -> ready on attempt 3", + "seconds": 12.8, + "observed": [ + "retry delays [1.12, 3.15] (windows [1,2] and [2,4] s)" + ] + }, + { + "scenario": "exhausted_then_replay", + "what": "5 transient failures -> failed retry_exhausted (registry) -> explicit replay -> ready; replay of ready refused", + "seconds": 38.3, + "observed": [ + "failed after 5 attempts, registry entry present", + "replayed job 5ff64075-edfa-4472-afed-a003bdbb601d (attempt 6 next)", + "replay of ready: refused: job is done (attachment ready); replay applies only to failed jobs" + ] + }, + { + "scenario": "long_running_not_hung", + "what": "a 30 s step longer than the 20 s lease: heartbeat renews the lease -> no second attempt", + "seconds": 37.4, + "observed": [ + "30 s processing with 20 s lease: attempt=1, no lease_expired" + ] + }, + { + "scenario": "duplicate", + "what": "3 raw duplicates of an already completed job: ack as no-op, nothing rewritten", + "seconds": 12.4, + "observed": [ + "3 duplicates -> 3 duplicate_noop, updated_at unchanged" + ] + }, + { + "scenario": "reorder", + "what": "relay stopped; deliveries arrive out of order: attempt 1, then 1 again and 4, then the outbox backlog", + "seconds": 20.2, + "observed": [ + "events: ['accepted', 'claimed', 'fault_injected', 'retry_scheduled', 'stale_attempt_noop', 'stale_attempt_noop', 'stale_attempt_noop', 'claimed', 'ready']" + ] + }, + { + "scenario": "new_version", + "what": "v1 ready -> reprocess v2 -> v2 published; replaying a v1 message afterwards changes nothing", + "seconds": 11.9, + "observed": [ + "v1 optimized 1090d84ed841.. -> v2 f2878c4a9d89..; stale v1 message: duplicate_noop" + ] + }, + { + "scenario": "stale_version_race", + "what": "v1 attempt is slow; v2 commits first; v1 then tries to commit -> superseded, v2 result kept", + "seconds": 16.3, + "observed": [ + "commit order: [('ready', 'v2: thumbnail,crop,optimized'), ('superseded', 'published v2 is newer than v1')]; v1 objects are unreferenced leftovers for the sweeper" + ] + }, + { + "scenario": "poison_message", + "what": "a message that violates jobs.schema.json goes to the DLQ instead of looping", + "seconds": 6.4, + "observed": [ + "DLQ 1 -> 2" + ] + } + ], + "final_stats": { + "jobs": { + "superseded": 2, + "failed": 6, + "done": 47 + }, + "outbox_unpublished": 0, + "queue_ready_messages": 0, + "queue_consumers": 1, + "dlq_messages": 2 + } +} \ No newline at end of file diff --git a/evidence/faults.txt b/evidence/faults.txt new file mode 100644 index 00000000..e02e1fd7 --- /dev/null +++ b/evidence/faults.txt @@ -0,0 +1,53 @@ +# fault injection suite (levels 3-4, track 5.2) +# date: 2026-11-10T13:42:20Z, FAULT_INJECTION=1, WORKER_REPLICAS=1, WORKER_CONCURRENCY=2, LEASE_SECONDS=20 +# command: python3 student-tests/faults_test.py --output evidence/faults.json + +--- worker_down: worker stopped -> upload 202 -> message -> API + broker restart -> worker start -> ready + worker: messenger-lab5-worker-1 exited Exited (0) Less than a second ago + queue before broker restart: {"jobs": {"superseded": 1, "failed": 6, "done": 33, "queued": 1}, "outbox_unpublished": 0, "queue_ready_messages": 1, "queue_consumers": 0, "dlq_messages": 1} + queue after broker restart (volume kept): {"jobs": {"superseded": 1, "failed": 6, "done": 33, "queued": 1}, "outbox_unpublished": 0, "queue_ready_messages": 1, "queue_consumers": 0, "dlq_messages": 1} + attachment 1f804983-862e-4fcc-b3ca-ee2af8564aba ready after worker start; thumbnail 256x160 +PASS worker_down (31.1s) +--- outbox_gap: COMMIT done, relay down (= crash between COMMIT and publish) -> relay restart publishes -> ready + relay down: outbox [{'id': 60, 'attempt': '1', 'published': False, 'publish_count': 0}], attachment queued + after relay start: ready, events ['accepted', 'claimed', 'ready'] +PASS outbox_gap (15.8s) +--- relay_crash_after_confirm: relay dies after the publisher confirm but before marking the row -> duplicate publish -> worker no-op + events: ['accepted', 'fault_injected', 'claimed', 'ready', 'duplicate_noop'] +PASS relay_crash_after_confirm (7.6s) +--- crash_after_claim: crash before any side effect: lease expires, reaper re-enqueues attempt 2 + worker.after_claim: ready in 23.9s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2] +PASS crash_after_claim (28.5s) +--- crash_after_s3: crash after writing S3 objects, before COMMIT: retry overwrites the same keys, one set of variants + worker.after_s3_before_commit: ready in 24.2s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2] +PASS crash_after_s3 (29.0s) +--- crash_after_commit: crash after COMMIT, before ack: redelivery sees done -> ack as no-op, no second attempt + worker.after_commit_before_ack: ready in 1.7s; events ['accepted', 'claimed', 'ready', 'fault_injected', 'duplicate_noop']; variant attempts [1] +PASS crash_after_commit (6.6s) +--- transient_backoff: two transient failures -> retries with exponential backoff + jitter -> ready on attempt 3 + retry delays [1.12, 3.15] (windows [1,2] and [2,4] s) +PASS transient_backoff (12.8s) +--- exhausted_then_replay: 5 transient failures -> failed retry_exhausted (registry) -> explicit replay -> ready; replay of ready refused + failed after 5 attempts, registry entry present + replayed job 5ff64075-edfa-4472-afed-a003bdbb601d (attempt 6 next) + replay of ready: refused: job is done (attachment ready); replay applies only to failed jobs +PASS exhausted_then_replay (38.3s) +--- long_running_not_hung: a 30 s step longer than the 20 s lease: heartbeat renews the lease -> no second attempt + 30 s processing with 20 s lease: attempt=1, no lease_expired +PASS long_running_not_hung (37.4s) +--- duplicate: 3 raw duplicates of an already completed job: ack as no-op, nothing rewritten + 3 duplicates -> 3 duplicate_noop, updated_at unchanged +PASS duplicate (12.4s) +--- reorder: relay stopped; deliveries arrive out of order: attempt 1, then 1 again and 4, then the outbox backlog + events: ['accepted', 'claimed', 'fault_injected', 'retry_scheduled', 'stale_attempt_noop', 'stale_attempt_noop', 'stale_attempt_noop', 'claimed', 'ready'] +PASS reorder (20.2s) +--- new_version: v1 ready -> reprocess v2 -> v2 published; replaying a v1 message afterwards changes nothing + v1 optimized 1090d84ed841.. -> v2 f2878c4a9d89..; stale v1 message: duplicate_noop +PASS new_version (11.9s) +--- stale_version_race: v1 attempt is slow; v2 commits first; v1 then tries to commit -> superseded, v2 result kept + commit order: [('ready', 'v2: thumbnail,crop,optimized'), ('superseded', 'published v2 is newer than v1')]; v1 objects are unreferenced leftovers for the sweeper +PASS stale_version_race (16.3s) +--- poison_message: a message that violates jobs.schema.json goes to the DLQ instead of looping + DLQ 1 -> 2 +PASS poison_message (6.4s) +PASS: 14 fault scenarios; final {"jobs": {"superseded": 2, "failed": 6, "done": 47}, "outbox_unpublished": 0, "queue_ready_messages": 0, "queue_consumers": 1, "dlq_messages": 2} diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..7e241301 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,9 @@ +# worker limits +# date: 2026-11-10T13:41:24Z, branch: solution/bvt2304-koshelev, host: Apple M3 8 CPU 16 GiB, Docker Desktop 28.0.1 +# command: python3 student-tests/limits_test.py + +big image: 105048 bytes compressed, 6000x6000 +PASS default limits (768 MiB, 60 s): ready in 1.0s, {'thumbnail': (256, 256), 'crop': (128, 128), 'optimized': (1280, 1280)} +PASS RLIMIT_AS 160 MiB: failed/image_too_large after 1 attempt +PASS JOB_TIMEOUT 0.3 s: child killed each attempt, 4 retries, then failed/retry_exhausted +PASS: worker limits diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..f05f59de --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.249, + "concurrency": 4, + "successful_requests": 3995, + "failed_requests": 4, + "error_ratio": 0.001, + "successful_rps": 261.986, + "successful_latency_ms": { + "p50": 3.331, + "p95": 6.927, + "p99": 60.411 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c03ad5de --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,33 @@ +# persistence: restart API, worker, relay and all stores (postgres, valkey, s3, rabbitmq) with volumes +# date: 2026-11-10T13:47:05Z +# command: make test-persistence ACTION_SCRIPT=scripts/restart.sh ; hook output below is from a separate run + +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +## hook run separately to show container states before/after: +NAME STATUS +messenger-lab5-api-1 Up 5 seconds (healthy) +messenger-lab5-api-2 Up 6 seconds (healthy) +messenger-lab5-lb-1 Up 5 seconds (healthy) +messenger-lab5-ops-1 Up 38 seconds (healthy) +messenger-lab5-postgres-1 Up 41 seconds (healthy) +messenger-lab5-rabbitmq-1 Up 41 seconds (healthy) +messenger-lab5-relay-1 Up 35 seconds (healthy) +messenger-lab5-s3-1 Up 41 seconds (healthy) +messenger-lab5-sweeper-1 Up 6 seconds (healthy) +messenger-lab5-valkey-1 Up 41 seconds (healthy) +messenger-lab5-worker-1 Up 6 seconds (healthy) +NAME STATUS +messenger-lab5-api-1 Up 5 seconds (healthy) +messenger-lab5-api-2 Up 5 seconds (healthy) +messenger-lab5-lb-1 Up 5 seconds (healthy) +messenger-lab5-ops-1 Up 37 seconds (healthy) +messenger-lab5-postgres-1 Up 41 seconds (healthy) +messenger-lab5-rabbitmq-1 Up 41 seconds (healthy) +messenger-lab5-relay-1 Up 35 seconds (healthy) +messenger-lab5-s3-1 Up 41 seconds (healthy) +messenger-lab5-sweeper-1 Up 5 seconds (healthy) +messenger-lab5-valkey-1 Up 41 seconds (healthy) +messenger-lab5-worker-1 Up 5 seconds (healthy) diff --git a/evidence/pipeline.txt b/evidence/pipeline.txt new file mode 100644 index 00000000..37863af1 --- /dev/null +++ b/evidence/pipeline.txt @@ -0,0 +1,12 @@ +# functional pipeline suite +# date: 2026-11-10T13:41:17Z, branch: solution/bvt2304-koshelev, host: Apple M3 8 CPU 16 GiB, Docker Desktop 28.0.1 +# command: python3 student-tests/pipeline_test.py + +PASS sample.png 320x200 -> thumbnail 256x160, crop 128x128, webp 320x200; original byte-identical 1.9s +PASS EXIF orientation applied, EXIF stripped from variants (thumbnail 170x256) 0.2s +PASS RGBA 64x40: no upscaling of thumbnail, alpha composited on white, crop upscaled 0.8s +PASS 2000x500 -> thumbnail 256x64, optimized fit 1280x320 0.2s +PASS PNG with valid CRCs but broken IDAT -> failed/invalid_image after 1 attempt; attach 409, content 409 1.7s +PASS synchronous limits before 202: pixels/bytes 413, type 415, filename 400 0.1s +PASS outsider gets 404 for metadata/original/variant; unknown variant 400 0.2s +PASS: 7 pipeline cases diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c29791ff --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# public smoke +# date: 2026-11-10T13:41:13Z, branch: solution/bvt2304-koshelev, host: Apple M3 8 CPU 16 GiB, Docker Desktop 28.0.1 +# command: make test + +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stop-one.txt b/evidence/stop-one.txt new file mode 100644 index 00000000..4fbf38d1 --- /dev/null +++ b/evidence/stop-one.txt @@ -0,0 +1,2 @@ +2026-11-10T13:49:45Z stopped messenger-lab5-api-1 (45db2233b0c2) instance=api-cf4d97a2 state=exited +2026-11-10T13:50:09Z stopped messenger-lab5-api-1 (45db2233b0c2) instance=api-3423c899 state=exited diff --git a/evidence/sweeper.txt b/evidence/sweeper.txt new file mode 100644 index 00000000..f7af715b --- /dev/null +++ b/evidence/sweeper.txt @@ -0,0 +1,26 @@ +# sweeper: staging/orphan retention (grace lowered to 60 s for the demo; default 900 s) +# date: 2026-11-10T13:50:46Z, attachment with a superseded v1 attempt: b25d5064-4db0-4a29-b9fa-fad0b112a60d + +## S3 keys before +[ + "attachments/b25d50644db04a29b9fafad0b112a60d/original", + "attachments/b25d50644db04a29b9fafad0b112a60d/v1/crop.jpg", + "attachments/b25d50644db04a29b9fafad0b112a60d/v1/optimized.webp", + "attachments/b25d50644db04a29b9fafad0b112a60d/v1/thumbnail.jpg", + "attachments/b25d50644db04a29b9fafad0b112a60d/v2/crop.jpg", + "attachments/b25d50644db04a29b9fafad0b112a60d/v2/optimized.webp", + "attachments/b25d50644db04a29b9fafad0b112a60d/v2/thumbnail.jpg" +] +## dry run +{"dry_run": true, "grace_seconds": 60, "retention_seconds": 86400, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_orphans": 0, "final_orphans": 6, "outbox_pruned": 0, "objects_scanned": 209, "duration_ms": 187.3} +## real run +{"dry_run": false, "grace_seconds": 60, "retention_seconds": 86400, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_orphans": 0, "final_orphans": 6, "outbox_pruned": 0, "objects_scanned": 209, "duration_ms": 208.3} +## repeat (idempotent) +{"dry_run": false, "grace_seconds": 60, "retention_seconds": 86400, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_orphans": 0, "final_orphans": 0, "outbox_pruned": 0, "objects_scanned": 203, "duration_ms": 125.5} +## S3 keys after +[ + "attachments/b25d50644db04a29b9fafad0b112a60d/original", + "attachments/b25d50644db04a29b9fafad0b112a60d/v2/crop.jpg", + "attachments/b25d50644db04a29b9fafad0b112a60d/v2/optimized.webp", + "attachments/b25d50644db04a29b9fafad0b112a60d/v2/thumbnail.jpg" +] diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..e510c5ad --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,52 @@ +global + log stdout format raw local0 info + maxconn 2048 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold other 2s + hold refused 2s + hold nx 2s + hold timeout 2s + hold obsolete 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 1s + timeout client 30s + # uploads up to 10 MiB + image decode; inactivity timeouts, not total request time + timeout server 30s + timeout http-request 10s + timeout queue 5s + timeout check 1s + retries 3 + option redispatch 1 + retry-on conn-failure empty-response + unique-id-format %[uuid()] + http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API replica failed\",\"request_id\":\"%[unique-id]\"}}" + http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no healthy API replica\",\"request_id\":\"%[unique-id]\"}}" + http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API replica timeout\",\"request_id\":\"%[unique-id]\"}}" + +frontend public + bind :8080 + default_backend api + +frontend monitor + bind 127.0.0.1:8405 + monitor-uri /lb-health + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 8 api:8080 diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..a46a618c --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,49 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); + +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..b58ca4a8 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id), + key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$') +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..a5efcfe9 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,49 @@ +-- lab4: message text may be empty when the message carries attachments (checked in the API). +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id); + +-- Metadata only; bytes live in the private S3 bucket under object_key (assigned by the server). +-- 'pending' = intent recorded before S3 PUT, invisible to the API; the sweeper removes stale ones. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + object_key text NOT NULL, + status text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id), + CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000) +); + +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +-- chat_id is repeated so that both composite FKs guarantee: an attachment is linked only within its own chat. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, position), + CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id), + CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/0004_jobs_outbox.sql b/migrations/0004_jobs_outbox.sql new file mode 100644 index 00000000..d460f85f --- /dev/null +++ b/migrations/0004_jobs_outbox.sql @@ -0,0 +1,94 @@ +-- lab5: asynchronous processing. Upload = staging object + attachment(queued) + job + outbox row in ONE transaction. + +-- Lab4 'pending' rows were invisible intents of an unfinished upload; they have no lab5 meaning. +DELETE FROM attachments WHERE status = 'pending'; +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; +ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')); +-- staging_key: private S3 prefix staging/ where the API puts the full original before 202. +-- object_key: final key of the published original (written by the worker, served only when status = ready). +-- pipeline_version: version whose variants are currently published (NULL until the first ready). +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD COLUMN staging_deleted_at timestamptz; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer; +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code CHECK (error_code IS NULL OR error_code ~ '^[a-z_]{1,64}$'); +DROP INDEX IF EXISTS attachments_pending_idx; +CREATE INDEX attachments_staging_idx ON attachments (status, updated_at) WHERE staging_deleted_at IS NULL; + +-- One logical unit of work = (attachment, pipeline_version). job_id never changes on redelivery or retry. +CREATE TABLE jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL DEFAULT 'queued', + attempt integer NOT NULL DEFAULT 0, -- attempts already started (fencing token together with job_id) + max_attempts integer NOT NULL, -- raised by an explicit operator replay + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_owner text, + lease_until timestamptz, + last_error text, + error_code text, + replays integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT jobs_effect_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT jobs_status CHECK (status IN ('queued', 'processing', 'done', 'failed', 'superseded')), + CONSTRAINT jobs_lease CHECK ((status = 'processing') = (lease_owner IS NOT NULL AND lease_until IS NOT NULL)), + CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts >= 1) +); +CREATE INDEX jobs_processing_lease_idx ON jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX jobs_queued_idx ON jobs (next_attempt_at) WHERE status = 'queued'; +CREATE INDEX jobs_failed_idx ON jobs (updated_at) WHERE status = 'failed'; + +-- Transactional outbox: written in the same transaction as the state change, published by the relay. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (job_id), + payload jsonb NOT NULL, -- exactly contracts/jobs.schema.json, no bytes/URLs/credentials + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- delayed retry (backoff + jitter) + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + publish_count integer NOT NULL DEFAULT 0 +); +CREATE INDEX outbox_unpublished_idx ON outbox (available_at, id) WHERE published_at IS NULL; + +-- Effect key (attachment_id, pipeline_version, name): one row per variant, never a second set. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + job_id uuid NOT NULL REFERENCES jobs (job_id), + attempt integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Audit/observability log: claims, retries, duplicates, replays. Never contains secrets or bytes. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid, + attachment_id uuid, + event text NOT NULL, + attempt integer, + actor text NOT NULL, + detail text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX job_events_attachment_idx ON job_events (attachment_id, id); + +-- Fault injection for the track-5 tests; consulted only when FAULT_INJECTION=1 in the worker/relay environment. +CREATE TABLE fault_injections ( + id bigserial PRIMARY KEY, + point text NOT NULL, + action text NOT NULL, + attachment_id uuid, + remaining integer NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..9774ad8a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.142.2 +uvicorn[standard]==0.54.0 +pydantic==2.13.5 +asyncpg==0.32.0 +redis==8.1.0 +argon2-cffi==25.1.0 +boto3==1.43.120 +pillow==12.0.0 +aio-pika==9.5.8 diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..f77387ef --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +mkdir -p secrets + +gen() { python3 -c 'import secrets; print(secrets.token_hex(24))'; } +keyid() { python3 -c 'import secrets,string; a=string.ascii_uppercase+string.digits; print("".join(secrets.choice(a) for _ in range(20)))'; } +once() { [ -f "secrets/$1" ] || { "$2" > "secrets/$1"; echo "created secrets/$1"; }; } + +once postgres_password gen +PG_PASS=$(cat secrets/postgres_password) +printf 'postgresql://messenger:%s@postgres:5432/messenger\n' "$PG_PASS" > secrets/database_url + +if [ ! -f secrets/valkey.conf ]; then + { + printf 'requirepass %s\n' "$(gen)" + printf 'save ""\nappendonly no\n' + printf 'maxmemory 64mb\nmaxmemory-policy volatile-lru\n' + } > secrets/valkey.conf + echo "created secrets/valkey.conf" +fi +VK_PASS=$(sed -n 's/^requirepass //p' secrets/valkey.conf) +printf 'redis://:%s@valkey:6379/0\n' "$VK_PASS" > secrets/cache_url + +# S3 (SeaweedFS): admin identity only for bucket bootstrap; API identity limited to Read/Write/List of one bucket. +BUCKET=${S3_BUCKET:-messenger-attachments} +once s3_admin_access_key_id keyid +once s3_admin_secret_access_key gen +once s3_access_key_id keyid +once s3_secret_access_key gen +python3 - "$BUCKET" > secrets/seaweedfs_s3.json <<'PY' +import json, sys +r = lambda n: open("secrets/" + n).read().strip() +b = sys.argv[1] +print(json.dumps({"identities": [ + {"name": "bootstrap-admin", + "credentials": [{"accessKey": r("s3_admin_access_key_id"), "secretKey": r("s3_admin_secret_access_key")}], + "actions": ["Admin", "Read", "List", "Write"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": r("s3_access_key_id"), "secretKey": r("s3_secret_access_key")}], + "actions": ["Read:" + b, "Write:" + b, "List:" + b]}, +]}, indent=1)) +PY + +# RabbitMQ: the default user is defined in a conf.d file mounted as a secret (no password in compose/env). +once rabbitmq_password gen +RMQ_PASS=$(cat secrets/rabbitmq_password) +{ + printf 'default_user = messenger\n' + printf 'default_pass = %s\n' "$RMQ_PASS" + printf 'default_vhost = /\n' + printf 'loopback_users.guest = true\n' + printf 'vm_memory_high_watermark.absolute = 300MiB\n' + printf 'disk_free_limit.absolute = 256MB\n' +} > secrets/rabbitmq.conf +printf 'amqp://messenger:%s@rabbitmq:5672/\n' "$RMQ_PASS" > secrets/broker_url + +chmod 644 secrets/* +echo "secrets ready in ./secrets (not tracked by git)" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..abb0dc35 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Restart hook for tests/resilience.py: API, worker, relay and ALL stores (PostgreSQL, Valkey, SeaweedFS, +# RabbitMQ) are stopped and started again with the same named volumes. Nothing is removed or re-seeded. +set -eu +cd "$(dirname "$0")/.." +docker compose ps --format 'table {{.Name}}\t{{.Status}}' +docker compose stop lb api worker relay sweeper ops +docker compose stop valkey postgres s3 rabbitmq +docker compose up -d --no-build --wait api lb worker relay sweeper ops +docker compose ps --format 'table {{.Name}}\t{{.Status}}' diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..4f8a3207 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-build --no-recreate --wait api +docker compose ps api --format 'table {{.Name}}\t{{.Status}}' diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 00000000..a65532b4 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-build --wait worker +docker compose ps worker --format 'table {{.Name}}\t{{.Status}}' diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..8d979ae3 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +log=evidence/stop-one.txt +for cid in $(docker compose ps -q api); do + alias=$(docker exec "$cid" cat /tmp/instance-id 2>/dev/null || true) + if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then + name=$(docker inspect -f '{{.Name}}' "$cid" | sed 's#^/##') + docker stop --time 8 "$cid" >/dev/null + state=$(docker inspect -f '{{.State.Status}}' "$cid") + printf '%s stopped %s (%s) instance=%s state=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$name" "$(printf %.12s "$cid")" "$alias" "$state" | tee -a "$log" + exit 0 + fi +done +echo "no running api container reports instance id $TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..5d726336 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Stops every worker container (the API, relay and broker keep running). Volumes are untouched. +set -eu +cd "$(dirname "$0")/.." +docker compose stop worker +docker compose ps worker --all --format 'table {{.Name}}\t{{.Status}}' diff --git a/student-tests/faults_test.py b/student-tests/faults_test.py new file mode 100644 index 00000000..b40b0a82 --- /dev/null +++ b/student-tests/faults_test.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +"""Automatic fault injection for the attachment pipeline (levels 3-4 and track 5.2 "proved idempotency"). + +Requires the stack started with FAULT_INJECTION=1 (see REPORT.md). Controls containers via `docker compose` +and arms faults via the operator CLI; never deletes volumes. Each scenario ends with the invariant check: + + I1 accepted work is not lost: every 202 attachment reaches its expected final state + I2 one result set: exactly one variant row per (attachment, version, variant) for the published version, + API metadata == DB rows, and bytes served by the API match the published sha256/size + I3 no duplicate effects: final S3 prefix == original + keys referenced by committed rows (+ explicitly + expected unreferenced leftovers of a superseded version) + I4 monotonic versions: 'ready' events never go back to an older pipeline_version; published version is the max + I5 nothing left behind: no job in processing, outbox fully published, staging removed after ready + + python3 student-tests/faults_test.py # all scenarios + python3 student-tests/faults_test.py --only duplicate,reorder +""" +import hashlib +import json +import time +import uuid +from urllib.parse import quote + +from stack import SAMPLE_PNG, arm, corrupt_png, dc, inspect, ops, stats, wait_for +from client import Client, Failure, arguments, check, run + +SCENARIOS = [] + + +def scenario(fn): + SCENARIOS.append(fn) + return fn + + +class Ctx: + def __init__(self, c, alice, bob, chat): + self.c, self.alice, self.bob, self.chat = c, alice, bob, chat + self.notes = [] + + def upload(self, data=SAMPLE_PNG, name="f.png"): + path = "/api/v1/chats/" + self.chat["id"] + "/attachments?filename=" + quote(name, safe="") + att = self.c.call("POST", path, 202, actor=self.alice, raw=data, headers={"Content-Type": "image/png"}, + schema="Attachment")["json"] + check(att["status"] == "queued", "upload must answer 202 queued") + return att + + def get(self, att_id): + return self.c.call("GET", "/api/v1/attachments/" + att_id, actor=self.bob, schema="Attachment")["json"] + + def wait(self, att_id, status="ready", timeout=90): + def done(): + a = self.get(att_id) + if a["status"] in ("ready", "failed"): + return a + return None + a = wait_for(done, timeout, "final state of " + att_id, 0.3) + check(a["status"] == status, "expected %s, got %s (%s)" % (status, a["status"], a["error_code"])) + return a + + def message(self, text, ids=None): + return self.c.message(self.alice, self.chat, text, attachments=ids)["json"] + + def note(self, text): + self.notes.append(text) + print(" ", text) + + +def events(info, name=None): + return [e for e in info["events"] if name is None or e["event"] == name] + + +def check_invariants(ctx, att_id, expect_version=1, leftovers=()): + info = inspect(att_id) + att = ctx.get(att_id) + a = info["attachment"] + check(att["status"] == a["status"] == "ready", "I1: not ready") + published = a["pipeline_version"] + check(published == expect_version, "I4: published v%s, expected v%s" % (published, expect_version)) + rows = [v for v in info["variants"] if v["pipeline_version"] == published] + names = sorted(v["name"] for v in rows) + check(names == ["crop", "optimized", "thumbnail"], "I2: variant rows %s" % names) + api = {v["name"]: v for v in att["variants"]} + for v in rows: + m = api[v["name"]] + check((m["sha256"], m["size_bytes"], m["width"], m["height"]) == (v["sha256"], v["size_bytes"], v["width"], v["height"]), + "I2: API metadata differs from DB row") + d = ctx.c.call("GET", "/api/v1/attachments/%s/content?variant=%s" % (att_id, v["name"]), actor=ctx.bob) + check(hashlib.sha256(d["bytes"]).hexdigest() == v["sha256"], "I2: served bytes differ from published sha256") + original = ctx.c.call("GET", "/api/v1/attachments/%s/content" % att_id, actor=ctx.bob)["bytes"] + check(hashlib.sha256(original).hexdigest() == a["sha256"], "I2: original changed") + expected = {"attachments/%s/original" % uuid.UUID(att_id).hex} | {v["object_key"] for v in info["variants"]} | set(leftovers) + check(set(info["s3_final_keys"]) == expected, "I3: S3 keys %s != %s" % (info["s3_final_keys"], sorted(expected))) + readies = [int(e["detail"].split(":")[0][1:]) for e in events(info, "ready")] + check(readies == sorted(readies) and readies[-1] == published, "I4: ready history %s" % readies) + check(all(j["status"] != "processing" for j in info["jobs"]), "I5: job left in processing") + check(all(o["published"] for o in info["outbox"]), "I5: unpublished outbox rows") + check(info["s3_staging_keys"] == [], "I5: staging not removed") + return info + + +def compose_ps(service): + return dc("ps", service, "--all", "--format", "{{.Name}} {{.State}} {{.Status}}").stdout.strip() + + +# ============================================================ level 3 +@scenario +def worker_down(ctx): + """worker stopped -> upload 202 -> message -> API + broker restart -> worker start -> ready""" + dc("stop", "worker") + ctx.note("worker: " + compose_ps("worker")) + att = ctx.upload(name="while-worker-down.png") + msg = ctx.message("файл, пока worker остановлен", [att["id"]]) + check(msg["attachment_ids"] == [att["id"]], "queued attachment must be linkable") + ctx.c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=ctx.bob) + other = ctx.message("другие сообщения работают") + check(other["text"], "messaging works while the worker is down") + time.sleep(2) + s = stats() + check(s["queue_ready_messages"] >= 1 and s["queue_consumers"] == 0, "message must wait in the queue: %s" % s) + ctx.note("queue before broker restart: %s" % json.dumps(s)) + dc("restart", "rabbitmq") + dc("up", "-d", "--wait", "--no-build", "rabbitmq", "relay") + s = wait_for(lambda: (lambda x: x if x["queue_ready_messages"] >= 1 else None)(stats()), 60, "queue after restart") + ctx.note("queue after broker restart (volume kept): %s" % json.dumps(s)) + dc("restart", "api") + dc("up", "-d", "--wait", "--no-build", "api", "lb") + check(ctx.get(att["id"])["status"] == "queued", "still queued after API restart") + dc("up", "-d", "--wait", "--no-build", "worker") + ready = ctx.wait(att["id"]) + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail size") + check_invariants(ctx, att["id"]) + ctx.note("attachment %s ready after worker start; thumbnail 256x160" % att["id"]) + + +# ============================================================ level 4: outbox, crashes, retries, lease, replay +@scenario +def outbox_gap(ctx): + """COMMIT done, relay down (= crash between COMMIT and publish) -> relay restart publishes -> ready""" + dc("stop", "relay") + att = ctx.upload(name="outbox-gap.png") + time.sleep(1.5) + info = inspect(att["id"]) + check([o["published"] for o in info["outbox"]] == [False], "outbox row must be pending while relay is down") + check(ctx.get(att["id"])["status"] == "queued", "queued") + ctx.note("relay down: outbox %s, attachment queued" % info["outbox"]) + dc("up", "-d", "--wait", "--no-build", "relay") + ctx.wait(att["id"]) + info = check_invariants(ctx, att["id"]) + ctx.note("after relay start: ready, events %s" % [e["event"] for e in info["events"]]) + + +@scenario +def relay_crash_after_confirm(ctx): + """relay dies after the publisher confirm but before marking the row -> duplicate publish -> worker no-op""" + arm("relay.after_confirm_before_mark", "crash") + att = ctx.upload(name="relay-crash.png") + ctx.wait(att["id"]) + info = wait_for(lambda: (lambda i: i if events(i, "duplicate_noop") else None)(inspect(att["id"])), 60, + "duplicate delivery") + info = check_invariants(ctx, att["id"]) + check(len(events(info, "claimed")) == 1, "the duplicate must not start a second attempt") + ctx.note("events: %s" % [e["event"] for e in info["events"]]) + + +def crash_scenario(ctx, point, name): + arm(point, "crash") + att = ctx.upload(name=name) + started = time.monotonic() + ctx.wait(att["id"], timeout=120) + info = check_invariants(ctx, att["id"]) + ctx.note("%s: ready in %.1fs; events %s; variant attempts %s" % ( + point, time.monotonic() - started, [e["event"] for e in info["events"]], + sorted({v["attempt"] for v in info["variants"]}))) + return info + + +@scenario +def crash_after_claim(ctx): + """crash before any side effect: lease expires, reaper re-enqueues attempt 2""" + info = crash_scenario(ctx, "worker.after_claim", "crash-claim.png") + check(events(info, "lease_expired") and info["jobs"][0]["attempt"] == 2, "recovered via lease expiry") + + +@scenario +def crash_after_s3(ctx): + """crash after writing S3 objects, before COMMIT: retry overwrites the same keys, one set of variants""" + info = crash_scenario(ctx, "worker.after_s3_before_commit", "crash-s3.png") + check(events(info, "lease_expired") and {v["attempt"] for v in info["variants"]} == {2}, "result of attempt 2") + + +@scenario +def crash_after_commit(ctx): + """crash after COMMIT, before ack: redelivery sees done -> ack as no-op, no second attempt""" + info = crash_scenario(ctx, "worker.after_commit_before_ack", "crash-ack.png") + check(events(info, "duplicate_noop"), "redelivered message must be a no-op") + check(info["jobs"][0]["attempt"] == 1 and len(events(info, "claimed")) == 1, "single attempt") + + +@scenario +def transient_backoff(ctx): + """two transient failures -> retries with exponential backoff + jitter -> ready on attempt 3""" + arm("worker.before_transform", "transient", count=2) + att = ctx.upload(name="transient.png") + ctx.wait(att["id"]) + info = check_invariants(ctx, att["id"]) + delays = [float(e["detail"].rsplit(" in ", 1)[1].rstrip("s")) for e in events(info, "retry_scheduled")] + check(len(delays) == 2 and 1.0 <= delays[0] <= 2.0 and 2.0 <= delays[1] <= 4.0, "backoff windows %s" % delays) + check(info["jobs"][0]["attempt"] == 3, "ready on attempt 3") + ctx.note("retry delays %s (windows [1,2] and [2,4] s)" % delays) + + +@scenario +def exhausted_then_replay(ctx): + """5 transient failures -> failed retry_exhausted (registry) -> explicit replay -> ready; replay of ready refused""" + arm("worker.before_transform", "transient", count=5) + att = ctx.upload(name="exhausted.png") + failed = ctx.wait(att["id"], "failed", timeout=120) + check(failed["error_code"] == "retry_exhausted", "error_code retry_exhausted") + registry = json.loads(ops("failed").stdout) + check(any(r["attachment_id"] == att["id"] and r["attempt"] == 5 for r in registry), "failed registry entry") + ctx.note("failed after 5 attempts, registry entry present") + out = ops("replay", att["id"], "--actor", "teacher-demo") + ctx.note(out.stdout.strip()) + ctx.wait(att["id"]) + info = check_invariants(ctx, att["id"]) + refused = ops("replay", att["id"], "--actor", "teacher-demo", check_rc=False) + check(refused.returncode == 3, "replay of a ready job must be refused") + info2 = inspect(att["id"]) + check(len(events(info2, "claimed")) == len(events(info, "claimed")), "refused replay ran nothing") + check(info2["jobs"][0]["attempt"] == 6 and info2["jobs"][0]["replays"] == 1, "attempt 6 after replay") + ctx.note("replay of ready: %s" % refused.stdout.strip()) + + +@scenario +def long_running_not_hung(ctx): + """a 30 s step longer than the 20 s lease: heartbeat renews the lease -> no second attempt""" + arm("worker.before_transform", "delay:30") + att = ctx.upload(name="slow.png") + ctx.wait(att["id"], timeout=90) + info = check_invariants(ctx, att["id"]) + check(not events(info, "lease_expired") and info["jobs"][0]["attempt"] == 1, "slow worker must keep its lease") + ctx.note("30 s processing with 20 s lease: attempt=1, no lease_expired") + + +# ============================================================ track 5.2: duplicates, reorder, versions +@scenario +def duplicate(ctx): + """3 raw duplicates of an already completed job: ack as no-op, nothing rewritten""" + att = ctx.upload(name="dup.png") + ctx.wait(att["id"]) + before = inspect(att["id"]) + for _ in range(3): + ops("republish", att["id"], "--attempt", "1") + after = wait_for(lambda: (lambda i: i if len(events(i, "duplicate_noop")) >= 3 else None)(inspect(att["id"])), 30, + "3 duplicate no-ops") + check(after["variants"] == before["variants"] and after["attachment"] == before["attachment"], "state changed") + check_invariants(ctx, att["id"]) + ctx.note("3 duplicates -> %d duplicate_noop, updated_at unchanged" % len(events(after, "duplicate_noop"))) + + +@scenario +def reorder(ctx): + """relay stopped; deliveries arrive out of order: attempt 1, then 1 again and 4, then the outbox backlog""" + dc("stop", "relay") # the outbox keeps its rows; we deliver raw messages in our own order + arm("worker.before_transform", "transient") + att = ctx.upload(name="reorder.png") + ops("republish", att["id"], "--attempt", "1") # attempt 1 runs and fails transiently -> attempt 2 in outbox + wait_for(lambda: events(inspect(att["id"]), "retry_scheduled"), 30, "first transient failure") + ops("republish", att["id"], "--attempt", "1") # stale: attempt 1 already happened + ops("republish", att["id"], "--attempt", "4") # "future": not the next attempt + wait_for(lambda: len(events(inspect(att["id"]), "stale_attempt_noop")) >= 2, 30, "stale no-ops") + check(ctx.get(att["id"])["status"] == "queued", "stale messages must not start an attempt") + dc("up", "-d", "--wait", "--no-build", "relay") # backlog: original attempt-1 row (stale now) + attempt 2 + ctx.wait(att["id"]) + info = check_invariants(ctx, att["id"]) + check(info["jobs"][0]["attempt"] == 2 and len(events(info, "claimed")) == 2, "exactly two real attempts") + check(len(events(info, "stale_attempt_noop")) >= 3, "late outbox copy of attempt 1 must be stale too") + ctx.note("events: %s" % [e["event"] for e in info["events"]]) + + +@scenario +def new_version(ctx): + """v1 ready -> reprocess v2 -> v2 published; replaying a v1 message afterwards changes nothing""" + att = ctx.upload(name="versions.png") + v1 = ctx.wait(att["id"]) + ops("reprocess", att["id"], "--pipeline-version", "2") + wait_for(lambda: inspect(att["id"])["attachment"]["pipeline_version"] == 2, 60, "v2 published") + v2 = ctx.get(att["id"]) + check(v2["status"] == "ready", "ready never goes back to processing during reprocess") + opt1 = [v for v in v1["variants"] if v["name"] == "optimized"][0]["sha256"] + opt2 = [v for v in v2["variants"] if v["name"] == "optimized"][0]["sha256"] + check(opt1 != opt2, "v2 must publish different optimized bytes") + ops("republish", att["id"], "--pipeline-version", "1", "--attempt", "1") + info = wait_for(lambda: (lambda i: i if events(i, "duplicate_noop") else None)(inspect(att["id"])), 30, "v1 no-op") + check_invariants(ctx, att["id"], expect_version=2) + ctx.note("v1 optimized %s.. -> v2 %s..; stale v1 message: duplicate_noop" % (opt1[:12], opt2[:12])) + refused = ops("reprocess", att["id"], "--pipeline-version", "1", check_rc=False) + check(refused.returncode == 3, "downgrade must be refused") + + +@scenario +def stale_version_race(ctx): + """v1 attempt is slow; v2 commits first; v1 then tries to commit -> superseded, v2 result kept""" + arm("worker.after_s3_before_commit", "delay:8") + att = ctx.upload(name="race.png") + wait_for(lambda: events(inspect(att["id"]), "fault_injected"), 30, "v1 inside the delay window") + ops("reprocess", att["id"], "--pipeline-version", "2") + info = wait_for(lambda: (lambda i: i if any(j["status"] == "superseded" for j in i["jobs"]) else None)(inspect(att["id"])), + 60, "v1 superseded") + v1_keys = ["attachments/%s/v1/%s" % (uuid.UUID(att["id"]).hex, n) for n in ("crop.jpg", "optimized.webp", "thumbnail.jpg")] + check_invariants(ctx, att["id"], expect_version=2, leftovers=v1_keys) + check(all(v["pipeline_version"] == 2 for v in info["variants"]), "no v1 rows committed") + order = [(e["event"], e["detail"]) for e in info["events"] if e["event"] in ("ready", "superseded")] + ctx.note("commit order: %s; v1 objects are unreferenced leftovers for the sweeper" % order) + + +@scenario +def poison_message(ctx): + """a message that violates jobs.schema.json goes to the DLQ instead of looping""" + before = stats()["dlq_messages"] + code = ("import asyncio,aio_pika;from app import config\n" + "async def m():\n" + " c=await aio_pika.connect(config.BROKER_URL);ch=await c.channel(publisher_confirms=True)\n" + " ex=await ch.get_exchange(config.BROKER_EXCHANGE)\n" + " await ex.publish(aio_pika.Message(b'{\"job_id\":\"x\",\"url\":\"http://evil\"}',delivery_mode=2),routing_key=config.BROKER_ROUTING_KEY)\n" + " await c.close()\n" + "asyncio.run(m())") + dc("exec", "-T", "ops", "python", "-c", code) + after = wait_for(lambda: (lambda s: s if s["dlq_messages"] == before + 1 else None)(stats()), 30, "DLQ") + ctx.note("DLQ %d -> %d" % (before, after["dlq_messages"])) + + +def main(): + def configure(p): + p.add_argument("--only", default="") + p.add_argument("--output", default=None) + args = arguments("Fault injection for the attachment pipeline", configure) + c = Client(args.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + ctx = Ctx(c, alice, bob, chat) + ops("fault", "clear") + only = {x for x in args.only.split(",") if x} + report = [] + for fn in SCENARIOS: + if only and fn.__name__ not in only: + continue + print("--- %s: %s" % (fn.__name__, fn.__doc__.strip())) + ctx.notes = [] + started = time.monotonic() + try: + fn(ctx) + finally: + ops("fault", "clear") + dc("up", "-d", "--wait", "--no-build", "worker", "relay", "api", "lb", "rabbitmq") + took = time.monotonic() - started + print("PASS %s (%.1fs)" % (fn.__name__, took)) + report.append({"scenario": fn.__name__, "what": fn.__doc__.strip(), "seconds": round(took, 1), "observed": ctx.notes}) + final = stats() + check(final["outbox_unpublished"] == 0 and final["jobs"].get("processing", 0) == 0, "global I5: %s" % final) + print("PASS: %d fault scenarios; final %s" % (len(report), json.dumps(final))) + if args.output: + with open(args.output, "w", encoding="utf-8") as fh: + json.dump({"scenarios": report, "final_stats": final}, fh, ensure_ascii=False, indent=1) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/fixtures/alpha.png b/student-tests/fixtures/alpha.png new file mode 100644 index 0000000000000000000000000000000000000000..0a49b491f991edeb771a290b4194077571d5038c GIT binary patch literal 170 zcmeAS@N?(olHy`uVBq!ia0vp^4nVBI!3HG#&i#1_qzXM<978JRyuIKkc))?D;lPtA zVNr9xrR2ybFtp7r-#5`_b5Q+~Kl~;R>={xEm`!*uFwSCo!QjhW(y)xt%HcAD?1Ib& dz6&HsbN75=3V0gU9G120;#n6O1~{j7khlf{e_9jQ@`?$OFB@$_NG^ zuYdp}6VQ!p>>Qk2+&~3e1sIqZnVFebm_e=v%GLtq8CV2ag%k}P*@OcV*_8@Kj2b5{ zECr+Nabot8FYu9hwy!G(W<0ns_J%91?)yGetzkL1n{m0K=Ab&9e;|~U~ z`b!Y#FD4ciW)^mkzZjXyK^_-mVO2C_6LJh>Pb?HxGHT=yahkYr<3UbkKb$@|Xn~>=_=vSk(1r!hZ&*LtPpRm_T^- zz5fiWT0a0TC3yvZRB2pekB~BC~Dk6%6 zB4WcRs6hlHA|QwaM5QXd2&jZm5=h?LhqLaC?0h?A_uIYs?z_pmfA0Iw`O7;<9nu1( zt+aQt2N(MqmdNXON1O`@KG9?3w5L?%rff zAcw=q<1olmKt}ykz$dv(ZQH(Mr~m$--~%BCLk}G}dMuiG{3q7wGiT$^C7e&Zkd}Tq z<4We$YuB@L?%uo4%YBgdr08jJN$In)@|QKQUcafWt8Zv+Ywzgn>h5{pD;yLJ4Ude9 zC1Vq~FhKriSm^a*{a-onGm_!V)f zcM4VX&D{nFTlQ6}Y8fnQH55)j`;6?50b~3nWd8*A3$6yBE{{QjCyxUZAi8dx#@}I! zHBMuJ|7_+pi`T-75RkGMUS)@X;#x&(i^rW(zE75Fcr9)?!Or#m2hA=1U&1w zdW#1W_%W28DF`^@f&gI<^wmkjH}p_<8tCGM@T$$^J0I38mM;!6Y+ozl&eRZT{M&#AS8QxtwLNs{rSj1Ih58P99lAF& zvA%ml32zyd3#H*nd*fSOvm|~qecrq{?^)P3|NWA?O#yE$B&VvWgr1~UD&7V2WzxNC z%i@PGMXI$!(#HsBqRC1kKFK5CIReH*5U@uTtplvB`1fkStc#)zJ!{W`>Q+Fq8Lx*r z!%N#~5}jb~xH1g^>Dv5V1MaeM0<0k6XqGfT&gez6MvrC8?MWA_m&v`^*HJ!J z$yX5a;Z4v-Be>mm^e_TC5g>br0Gs@((QIR}UZFR=uw1 zWH2Om!4v$}?fmXyx>nE)rQEyKSsa%KF|2PZmhmbzDwM7EPF0+b>7G)RJg<$-_7mb? zT-g~b+==5EovQs-L%k?F_PIEDFOk8${oL|^&GjoL{$mhFLZ(HP>6d37Z{|rZJxUeQ zN{H?%-#omZ?UK8oa8HI)S$B8SesO#3+2<4%X(6N2Fu^7IJ<*EgfQb#Bd>t_T8 zc~(K&Hc3Jw&PH~GQCy1Fy?YVyY=k}~eAC@{E%G$4rTH0Aq7U{x15Rb8t2@Eg)GeVc zL0no+-u;66&KIrE@7Wao;hfRbtRqJlmNpEtj=Wx2_u$JrOx*jyiYl0r{-Jv8fKAc?ClzXCG@G}-r`PmRV#LmCb#~C^y$M=1b9*{A z9)(scnHGxO^qM7pKrOES7@o7rdVU(qM0p|!{`Y|@DQIPlWUDXao?PGP zK=11kxErNhZ;KyUmqbM~DUGw|=S-r4L$!~|)8=33E2<4yy(XbDJLXx#Wz8gdmgnfW$+6MsW23N0C#<%{VnD|s+T%{RaC{{Kma=06 z#-;~j{YK*9%~ZUnAbGwk6%RZsBGK1CTFLe)ehauztro*`X_4EteYEipr%l$I^yl_zrGZIN9V|c z6<7hU;-u9OH+5Gz0(MNn`Q6SDVvW1JX86$VIgz}})*4y5%+k*^EVWS`wLFxU9^)j- z2&f%e`s2CmOxDErzK=HkXz^AF?xF1=YGN*qB>T`p+GDiLg5LFo>Rk5Kh@W9iiXb>i z&EA&_X;Dk`-tHo<>+c!=G2wYs{OrHQDsP!(X)q7Ff3&cjHIgTuYk2GWmb1iq`iWX8 zn=RO7(uPNW9CxJ>M487YfzCi@Q zZxPVQK!Fu%Jo*fbHPWEgXwv`Dy@GMD!Nk*G&)OOK`f-IH+^Sc7f2nFtO`dI__(aP? zzAF8r+o-nhixXiCE2UedrS{}FdHr0SjEJq*>KgnHXnN5bGm7s%zwg+XO0J4ciCsRE z&`B~C+Zl(Q;A?D5H`!8MzgpQe*T}!@%$a1x7E2r6Ov>(afm5ss-Jd>m<~zW&0oxqR zr4@lLxBAD2piT+NnHr?cWaN8qogW<`C#$>FvWZ=}-j|)92!u6OSG>j6#hxK10Yv67 zA*53M_9ExUf^WsHH4nHcV{es6dx8Tz!YJP9yF@n*rK|G739S)!#Y=AIRD%dF||%j z*R=Y(tRiB=<*qd5%$lyl58v6a`{Ps9*IBh{ou z%U)(JT1skMdB;<0q<6)q-4#v-r)k#=II9nWHND2%=%frQE2pbEa1Kz;WzsRw+4!n zbY(}I0t)iksG2-h&_)I$*7?F+Kkc~a4-Q558{juqh(9n7)mAg zlBXh|%78)^X?*A}g(34Q1bl7fApP#sbOZ=uQB;xW1Kaup#_2M<*5)xRvmF(71?SPt zu0!O>W)t%D*93)j;-RbI)~MDzC50*|yd1JbD)<8|{z4*iWMnIv$ca7BB(Bq8tSmQRW}<&`ZeG*cWSDP82>j#lo5 z)q@-C-17K|u6_>^yY@yMQ?GC;3ERKsD%ahtwJFCWDU*?w(wWZl*w>b$v-UgV+KZ<* SJV9I9jFqXsaHh5^H97yxs_XgU}T15lwjnhr+8U^E?!whu %s" % a["error_code"]) + check(info["jobs"][0]["attempt"] == 1, "memory limit is permanent, no retries") + print("PASS RLIMIT_AS 160 MiB: failed/image_too_large after 1 attempt") + + worker(JOB_TIMEOUT_SECONDS=0.3, RETRY_BASE_SECONDS=0.5, RETRY_MAX_SECONDS=2) + a = final(upload()) + info = inspect(a["id"]) + check(a["status"] == "failed" and a["error_code"] == "retry_exhausted", "timeout -> %s" % a["error_code"]) + errors = [e["detail"].split(";")[0] for e in info["events"] if e["event"] == "retry_scheduled"] + check(len(errors) == 4 and all(x == "processing_timeout" for x in errors), "timeouts %s" % errors) + print("PASS JOB_TIMEOUT 0.3 s: child killed each attempt, %d retries, then failed/retry_exhausted" % len(errors)) + finally: + worker() + print("PASS: worker limits") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/pipeline_test.py b/student-tests/pipeline_test.py new file mode 100644 index 00000000..167536ea --- /dev/null +++ b/student-tests/pipeline_test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Functional suite for the asynchronous pipeline (levels 3-4): variants, EXIF orientation, alpha, limits, +permanent failures. Black box through the LB, plus the operator CLI for attempt counters.""" +import hashlib +import time +import uuid +from urllib.parse import quote + +from stack import (SAMPLE_PNG, corrupt_png, fixture, has_exif, inspect, jpeg_size, pixel, png_header_only, + webp_size) +from client import Client, arguments, check, run + +LIMIT = 10 * 1024 * 1024 + + +def main(): + args = arguments("Async attachment pipeline: variants and failures") + c = Client(args.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + results = [] + + def case(name, fn): + started = time.monotonic() + fn() + results.append(name) + print("PASS %-70s %.1fs" % (name, time.monotonic() - started)) + + def upload(data, ctype="image/png", filename="f.png", expected=202): + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(filename, safe="") + r = c.call("POST", path, expected, actor=alice, raw=data, headers={"Content-Type": ctype}, + schema="Attachment" if expected == 202 else None) + return r["json"] + + def wait_final(att, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + att = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + if att["status"] in ("ready", "failed"): + return att + time.sleep(0.2) + raise AssertionError("attachment did not reach a final state") + + def content(att, variant=None, expected=200, actor=bob): + q = "" if variant is None else "?variant=" + variant + return c.call("GET", "/api/v1/attachments/" + att["id"] + "/content" + q, expected, actor=actor) + + def variants(att): + return {v["name"]: v for v in att["variants"]} + + def verify_variant(att, name, ctype, size): + v = variants(att)[name] + d = content(att, name) + check(d["headers"].get_content_type() == ctype == v["content_type"], name + " content type") + check(len(d["bytes"]) == v["size_bytes"] and hashlib.sha256(d["bytes"]).hexdigest() == v["sha256"], name + " checksum") + real = jpeg_size(d["bytes"]) if ctype == "image/jpeg" else webp_size(d["bytes"]) + check(real == size == (v["width"], v["height"]), "%s: expected %s, metadata %sx%s, real %s" % ( + name, size, v["width"], v["height"], real)) + check(not has_exif(d["bytes"]), name + " must not carry EXIF") + return d["bytes"] + + def sample(): + a = upload(SAMPLE_PNG, filename="sample.png") + check(a["status"] == "queued" and a["variants"] == [] and a["error_code"] is None, "202 snapshot") + r = wait_final(a) + check(r["status"] == "ready" and r["error_code"] is None, "sample ready") + check([v["name"] for v in r["variants"]] == ["thumbnail", "crop", "optimized"], "exactly three variants") + verify_variant(r, "thumbnail", "image/jpeg", (256, 160)) + verify_variant(r, "crop", "image/jpeg", (128, 128)) + verify_variant(r, "optimized", "image/webp", (320, 200)) # no upscaling beyond the source + orig = content(r) + check(orig["bytes"] == SAMPLE_PNG and orig["headers"].get_content_type() == "image/png", "original unchanged") + check(r["sha256"] == hashlib.sha256(SAMPLE_PNG).hexdigest() and r["size_bytes"] == len(SAMPLE_PNG), "original metadata") + info = inspect(r["id"]) + check(info["jobs"][0]["attempt"] == 1 and info["s3_staging_keys"] == [], "one attempt, staging removed") + check(len(info["s3_final_keys"]) == 4, "original + 3 variants in final prefix") + case("sample.png 320x200 -> thumbnail 256x160, crop 128x128, webp 320x200; original byte-identical", sample) + + def exif(): + data = fixture("exif_rot6.jpg") + r = wait_final(upload(data, "image/jpeg", "rotated.jpg")) + check(r["status"] == "ready", "exif jpeg ready") + # stored 300x200 with Orientation=6 -> displayed 200x300; fit 256: 170x256 (floor of 200*256/300) + verify_variant(r, "thumbnail", "image/jpeg", (170, 256)) + verify_variant(r, "optimized", "image/webp", (200, 300)) + verify_variant(r, "crop", "image/jpeg", (128, 128)) + check(content(r)["bytes"] == data and has_exif(data), "original keeps its own EXIF byte-for-byte") + case("EXIF orientation applied, EXIF stripped from variants (thumbnail 170x256)", exif) + + def alpha(): + r = wait_final(upload(fixture("alpha.png"), filename="alpha.png")) + thumb = verify_variant(r, "thumbnail", "image/jpeg", (64, 40)) # small source is not upscaled + crop = verify_variant(r, "crop", "image/jpeg", (128, 128)) # crop may upscale + check(min(pixel(thumb, 56, 20)) >= 245, "transparent area must become white") + g = pixel(thumb, 8, 20) + check(g[1] > 120 and g[0] < 60 and g[2] < 60, "opaque green stays green") + check(min(pixel(crop, 120, 64)) >= 245, "crop: alpha on white") + case("RGBA 64x40: no upscaling of thumbnail, alpha composited on white, crop upscaled", alpha) + + def wide(): + r = wait_final(upload(fixture("wide.png"), filename="wide.png")) + verify_variant(r, "thumbnail", "image/jpeg", (256, 64)) + verify_variant(r, "optimized", "image/webp", (1280, 320)) + case("2000x500 -> thumbnail 256x64, optimized fit 1280x320", wide) + + def corrupt(): + a = upload(corrupt_png(), filename="broken.png") + r = wait_final(a) + check(r["status"] == "failed" and r["error_code"] == "invalid_image" and r["variants"] == [], "invalid_image") + info = inspect(r["id"]) + check(info["jobs"][0]["attempt"] == 1, "permanent error must not be retried") + check(not any(e["event"] == "retry_scheduled" for e in info["events"]), "no retry events") + content(r, None, 409) + content(r, "thumbnail", 409) + c.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 409, actor=alice, + body={"text": "", "attachment_ids": [r["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + failed = c.call("GET", "/api/v1/attachments/" + r["id"], actor=alice, schema="Attachment")["json"] + check(failed["error_code"] == "invalid_image", "error visible to members") + case("PNG with valid CRCs but broken IDAT -> failed/invalid_image after 1 attempt; attach 409, content 409", corrupt) + + def limits(): + upload(png_header_only(8000, 6000), filename="bomb.png", expected=413) # 48 MP, header only + upload(b"\x89PNG\r\n\x1a\n" + bytes(LIMIT), filename="big.png", expected=413) + upload(b"GIF89a" + bytes(100), "image/png", "x.png", expected=415) + upload(fixture("sample.jpg"), "image/png", "lie.png", expected=415) + upload(SAMPLE_PNG, "image/png", "a/b.png", expected=400) + case("synchronous limits before 202: pixels/bytes 413, type 415, filename 400", limits) + + def access(): + r = wait_final(upload(SAMPLE_PNG, filename="acl.png")) + for suffix in ("", "/content", "/content?variant=thumbnail"): + c.call("GET", "/api/v1/attachments/" + r["id"] + suffix, 404, actor=eve) + content(r, "bogus", 400) + case("outsider gets 404 for metadata/original/variant; unknown variant 400", access) + + print("PASS: %d pipeline cases" % len(results)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/stack.py b/student-tests/stack.py new file mode 100644 index 00000000..31744d3d --- /dev/null +++ b/student-tests/stack.py @@ -0,0 +1,119 @@ +"""Helpers for student tests: synthetic images (stdlib only), operator CLI and compose control.""" +import json +import os +import struct +import subprocess +import sys +import time +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Failure, check # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +SAMPLE_PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def fixture(name): + return (FIXTURES / name).read_bytes() + + +# ---------------------------------------------------------------- images +def _chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, row=None): + blank = bytes(width * 3) + comp = zlib.compressobj(6) + parts = [comp.compress(b"\x00" + (row(y) if row else blank)) for y in range(height)] + parts.append(comp.flush()) + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", b"".join(parts)) + _chunk(b"IEND", b"") + + +def png_header_only(width, height): + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(bytes(1000))) + _chunk(b"IEND", b"") + + +def corrupt_png(width=64, height=64): + """Valid signature, IHDR and chunk CRCs (passes the synchronous API checks), but IDAT is not zlib: + only a full decode in the worker can detect it -> failed / invalid_image, without retries.""" + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + garbage = bytes((i * 37 + 11) % 256 for i in range(4000)) + return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", garbage) + _chunk(b"IEND", b"") + + +def webp_size(data): + check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not a WebP RIFF container") + kind = data[12:16] + if kind == b"VP8X": + w = 1 + int.from_bytes(data[24:27], "little") + h = 1 + int.from_bytes(data[27:30], "little") + return w, h + if kind == b"VP8 ": + w, h = struct.unpack("> 14) & 0x3FFF) + 1 + raise Failure("unknown WebP chunk") + + +def has_exif(data): + return b"Exif\x00\x00" in data[:65536] + + +# ---------------------------------------------------------------- compose / operator CLI +def dc(*args, check_rc=True, timeout=180, input=None, env=None): + full_env = dict(os.environ, **env) if env else None + out = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, timeout=timeout, + input=input, env=full_env) + if check_rc and out.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args[:2]), out.stderr.strip()[-400:])) + return out + + +def ops(*args, check_rc=True): + return dc("exec", "-T", "ops", "python", "-m", "app.ops", *args, check_rc=check_rc) + + +def inspect(att_id): + return json.loads(ops("inspect", att_id).stdout) + + +def stats(): + return json.loads(ops("stats").stdout) + + +def arm(point, action, attachment=None, count=1): + args = ["fault", "arm", point, action, "--count", str(count)] + if attachment: + args += ["--attachment", attachment] + ops(*args) + + +def pixel(data, x, y): + """Decode one pixel of an image with Pillow inside the ops container (the host has only the stdlib).""" + code = ("import sys,io;from PIL import Image;im=Image.open(io.BytesIO(sys.stdin.buffer.read())).convert('RGB');" + "print(*im.getpixel((%d,%d)))" % (x, y)) + out = subprocess.run(["docker", "compose", "exec", "-T", "ops", "python", "-c", code], cwd=ROOT, + input=data, capture_output=True, timeout=60) + check(out.returncode == 0, "pixel probe failed") + return tuple(int(v) for v in out.stdout.split()) + + +def wait_for(predicate, timeout, what, interval=0.5): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = predicate() + if last: + return last + time.sleep(interval) + raise Failure("timeout waiting for " + what) -- 2.54.0 From 616866f115e2c50a0dc3c0a41db7884db1e90e0f Mon Sep 17 00:00:00 2001 From: Ilya <145450507+IIcon18@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:32:39 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=D0=92=D1=8B=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=9B=D0=A05=20=D0=BD=D0=B0=205=20(=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=BA=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DEMO.md | 358 +++++++++++++++++++++++++++++++++++++++++++++++++----- REPORT.md | 19 +-- 2 files changed, 337 insertions(+), 40 deletions(-) diff --git a/DEMO.md b/DEMO.md index d656d5dc..6d053a9c 100644 --- a/DEMO.md +++ b/DEMO.md @@ -1,52 +1,346 @@ -# Сценарий защиты — лабораторная 5 +# Демонстрация лабораторной 5 на защите -Стенд поднят (`make up`, в `.env` `FAULT_INJECTION=1`). Все команды — из корня репозитория. В консоль не выводятся токены и пароли. +Сценарий повторяет «Сценарий защиты» из README: smoke → worker остановлен (409, сообщения работают) → рестарт API и брокера с volumes → worker запущен → ready. Затем отказы уровня 4 (crash после side effect до ack, разрыв COMMIT/publish, replay failed) и трек 2. + +**Важно:** +- Все команды выполняются по порядку **в одном окне терминала** из корня репозитория: переменные `B`, `TA`, `TB`, `TE`, `CHAT`, `ATT` и функции `j`, `pg`, `ops`, `st`, `wt`, `ev`, `up` живут только в нём. +- Копируйте блоки целиком. В строках команд нет комментариев `#`: интерактивный zsh не считает их комментариями. +- Для шагов с отказами в `.env` должно быть `FAULT_INJECTION=1`. +- `rabbitmqctl list_queues` показывает счётчики с задержкой в несколько секунд; точные числа даёт `make stats` (опрашивает брокер напрямую). + +--- + +## 0. Запуск стенда + +```bash +cd ~/Desktop/вуз/ВНП/lab5/mtusi2026lab5 +``` + +```bash +./scripts/init-secrets.sh && ls secrets/ +``` + +```bash +grep -E '^(APP_PORT|FAULT_INJECTION|WORKER_REPLICAS)=' .env +``` + +```bash +docker compose --env-file .env config --quiet && docker compose --env-file .env up -d --build --wait +``` + +```bash +docker compose ps -a --format 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' +``` + +> **Сказать:** «Наружу опубликован только `lb` на 127.0.0.1:8080 и UI RabbitMQ на loopback. `api` ×2, `relay` и `worker` — отдельные процессы. `migrate` и `s3-init` — one-shot, завершились с кодом 0. Брокер — RabbitMQ 4.1 с volume `rabbitmqdata`». + +```bash +docker compose exec rabbitmq rabbitmqctl list_queues name type durable messages consumers +``` + +> «`attachments.process` — quorum queue (Raft-лог на диске), durable, один consumer — worker. `attachments.dlq` — очередь для сообщений, которые нельзя обработать». + +Вспомогательные переменные и функции (один раз): + +```bash +B=http://127.0.0.1:8080 +j() { python3 -c 'import sys,json;print(json.load(sys.stdin)[sys.argv[1]])' "$1"; } +pg() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; } +ops() { docker compose exec -T ops python -m app.ops "$@"; } +st() { curl -s $B/api/v1/attachments/$1 -H "Authorization: Bearer $TB" | python3 -c 'import sys,json;d=json.load(sys.stdin);print(d["status"],d["error_code"],[(v["name"],v["width"],v["height"]) for v in d["variants"]])'; } +wt() { for i in $(seq 1 120); do s=$(st $1); case "$s" in ready*|failed*) echo "$s"; return;; esac; sleep 1; done; echo "timeout: $s"; } +ev() { ops inspect $1 | python3 -c 'import sys,json;d=json.load(sys.stdin);[print(" ",e["event"],"attempt=%s"%e["attempt"],e["actor"],e["detail"] or "") for e in d["events"]];print("jobs:",[(x["pipeline_version"],x["status"],"attempt=%s"%x["attempt"]) for x in d["jobs"]]);print("s3:",d["s3_final_keys"]);print("staging:",d["s3_staging_keys"])'; } +up() { curl -s -X POST "$B/api/v1/chats/$CHAT/attachments?filename=$2" -H "Authorization: Bearer $TA" -H 'Content-Type: image/png' --data-binary @$1 | j id; } +``` ## 1. Публичный smoke -```sh -make test # 202 -> ready, оригинал побайтно, настоящий JPEG 256x160 -make stats # jobs по статусам, outbox, глубина очереди, DLQ +```bash +make test BASE_URL=$B ``` -## 2. Worker остановлен: приём продолжается, content = 409 - -```sh -make stop-worker -python3 student-tests/faults_test.py --only worker_down # автоматически: upload 202, сообщение с вложением, - # content 409, другие сообщения 201, рестарт RabbitMQ и API, - # старт worker -> ready (печатает ID вложения и глубину очереди) +```bash +make stats ``` -Вручную, если преподаватель просит «руками»: `make stop-worker` → загрузить `tests/fixtures/sample.png` по `examples/requests.http` → `GET …/content` = 409 → `docker compose restart rabbitmq api` → `make stats` (сообщение в очереди сохранилось) → `make start-worker` → `GET /attachments/{id}` = ready. +> «Smoke: 202 queued → опрос до ready → оригинал побайтно → настоящий JPEG 256×160 (тест читает размеры из байтов JPEG, а не из JSON). `stats` — задания по статусам, неопубликованный outbox, глубина очереди и DLQ». -## 3. Отказы уровня 4 +## 2. Пользователи, чат и одна загрузка «вживую» -```sh -python3 student-tests/faults_test.py --only outbox_gap,relay_crash_after_confirm # разрыв COMMIT/publish -python3 student-tests/faults_test.py --only crash_after_s3,crash_after_commit # crash после side effect до ack -python3 student-tests/faults_test.py --only transient_backoff,exhausted_then_replay # backoff+jitter, failed, replay -python3 student-tests/faults_test.py --only long_running_not_hung # lease + heartbeat -make failed # реестр failed -docker compose exec ops python -m app.ops inspect # история событий, ключи S3 +```bash +mkuser() { curl -s -X POST $B/api/v1/users -H 'Content-Type: application/json' -d "{\"username\":\"$1_$RANDOM\",\"display_name\":\"$1\",\"password\":\"Demo-password-123\"}"; } +login() { curl -s -X POST $B/api/v1/auth/sessions -u "$1:Demo-password-123" | j token; } +UA=$(mkuser alice); UB=$(mkuser bob); UE=$(mkuser eve) +A=$(echo "$UA" | j id); Bo=$(echo "$UB" | j id) +TA=$(login $(echo "$UA" | j username)); TB=$(login $(echo "$UB" | j username)); TE=$(login $(echo "$UE" | j username)) +CHAT=$(curl -s -X POST $B/api/v1/chats -H "Authorization: Bearer $TA" -H 'Content-Type: application/json' -d "{\"title\":\"Демо\",\"member_ids\":[\"$A\",\"$Bo\"]}" | j id); echo "$CHAT" ``` -Ручной replay: `docker compose exec ops python -m app.ops replay --actor <фамилия>`; на ready-вложении команда отвечает `refused` (код 3). - -## 4. Трек 2: идемпотентность - -```sh -python3 student-tests/faults_test.py --only duplicate,reorder,new_version,stale_version_race,poison_message +```bash +curl -s -i -X POST "$B/api/v1/chats/$CHAT/attachments?filename=sample.png" -H "Authorization: Bearer $TA" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png | tee /tmp/att.txt | sed -n '1p;$p' +ATT=$(tail -1 /tmp/att.txt | j id) ``` -Что показать в выводе: `duplicate_noop`, `stale_attempt_noop`, порядок `ready v2` → `superseded v1`, неизменные ключи S3 (инвариант I3). +> «Ответ **202**, `status=queued`, `variants=[]`. 202 отправляется только после того, как полный оригинал лёг в `staging/` в S3 (S3 сам сверил SHA-256) и **одной транзакцией** зафиксированы вложение, задание и строка outbox». + +```bash +wt $ATT +``` + +```bash +ev $ATT +``` + +> «События: `accepted` (API) → `claimed attempt=1` (worker взял аренду) → `ready`. В S3 ровно 4 ключа: оригинал и три варианта по детерминированным ключам `v1/…`. Staging уже удалён — только после COMMIT». + +```bash +curl -s "$B/api/v1/attachments/$ATT/content?variant=thumbnail" -H "Authorization: Bearer $TB" -o /tmp/thumb.jpg && python3 -c 'import sys;sys.path.insert(0,"tests");from image_probe import jpeg_size;print("thumbnail JPEG:",jpeg_size(open("/tmp/thumb.jpg","rb").read()))' +``` + +```bash +curl -s $B/api/v1/attachments/$ATT/content -H "Authorization: Bearer $TB" | shasum -a 256; shasum -a 256 tests/fixtures/sample.png +``` + +```bash +for s in "" /content "/content?variant=thumbnail"; do curl -s -o /dev/null -w "eve $s -> %{http_code}\n" "$B/api/v1/attachments/$ATT$s" -H "Authorization: Bearer $TE"; done +``` + +> «Оригинал побайтно совпадает, thumbnail действительно 256×160. Посторонний получает 404 на всё». + +## 3. Уровень 3: worker остановлен → рестарт API и брокера → worker запущен + +```bash +./scripts/stop-worker.sh +``` + +```bash +ATT=$(up tests/fixtures/sample.png worker-down.png); echo "$ATT"; st $ATT +``` + +```bash +curl -s -o /dev/null -w "content while queued -> %{http_code}\n" $B/api/v1/attachments/$ATT/content -H "Authorization: Bearer $TB" +``` + +```bash +curl -s -o /dev/null -w "message with queued attachment -> %{http_code}\n" -X POST $B/api/v1/chats/$CHAT/messages -H "Authorization: Bearer $TA" -H "Idempotency-Key: demo-$RANDOM" -H 'Content-Type: application/json' -d "{\"text\":\"\",\"attachment_ids\":[\"$ATT\"]}" +curl -s -o /dev/null -w "plain message -> %{http_code}\n" -X POST $B/api/v1/chats/$CHAT/messages -H "Authorization: Bearer $TA" -H "Idempotency-Key: demo-$RANDOM" -H 'Content-Type: application/json' -d '{"text":"сообщения работают без worker"}' +``` + +> «Upload принят (202), content — 409 (ещё не готово), сообщение с queued-вложением и обычное сообщение — 201. API не делает resize сам: обработка ждёт worker». + +```bash +make stats +``` + +> «`queue_ready_messages=1`, `queue_consumers=0`: сообщение ждёт в очереди, потребителей нет». + +```bash +docker compose restart rabbitmq && docker compose up -d --wait --no-build rabbitmq relay +``` + +```bash +make stats +``` + +> «Сообщение пережило рестарт брокера: quorum queue + persistent-сообщение + volume». + +```bash +docker compose restart api && docker compose up -d --wait --no-build api lb && st $ATT +``` + +> «API перезапущен — задание не потерялось, вложение всё ещё `queued`: всё состояние в PostgreSQL и S3, у API нет локального состояния». + +```bash +./scripts/start-worker.sh && wt $ATT +``` + +```bash +ev $ATT +``` + +> «Worker запущен — то же самое вложение стало ready. ID и порядок действий видны в событиях». + +## 4. Уровень 4: разрыв COMMIT → publish (transactional outbox) + +```bash +docker compose stop relay +``` + +```bash +ATT=$(up tests/fixtures/sample.png outbox.png); echo "$ATT"; sleep 2; st $ATT +``` + +```bash +pg "SELECT o.id, o.payload->>'attempt' AS attempt, o.published_at, o.publish_count FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id='$ATT'" +``` + +> «Транзакция зафиксирована, а публикации нет — именно окно «после COMMIT до publish». Намерение не потерялось: оно лежит в outbox». + +```bash +docker compose up -d --wait --no-build relay && wt $ATT +``` + +> «Relay поднялся, опубликовал строку с ожиданием publisher confirm, отметил `published_at` — задача дошла до ready без ручного создания задания». + +## 5. Уровень 4: crash worker после записи в S3, до COMMIT/ack + +```bash +ops fault arm worker.after_s3_before_commit crash +``` + +```bash +ATT=$(up tests/fixtures/sample.png crash.png); echo "$ATT"; sleep 3; docker compose ps worker +``` + +> «Worker записал объекты в S3 и умер (`os._exit(137)`, как SIGKILL) до COMMIT. Docker его перезапустил. Сообщение без ack вернулось в очередь». + +```bash +wt $ATT +``` + +```bash +ev $ATT +``` + +> «Redelivery увидела активную аренду мёртвого владельца → `lease_held_noop` (новый процесс не может знать, что старый умер). Через 20 с аренда истекла → `lease_expired` → retry → `claimed attempt=2` → `ready`. В S3 по-прежнему **ровно 4 ключа**: вторая попытка перезаписала те же ключи эффекта, второго набора нет». + +```bash +ops fault arm worker.after_commit_before_ack crash +``` + +```bash +ATT=$(up tests/fixtures/sample.png crash-ack.png); wt $ATT; sleep 4; ev $ATT +``` + +> «Здесь worker умер уже после COMMIT, но до ack. Повторная доставка видит `done` → `duplicate_noop` и ack. Попытка одна». + +## 6. Уровень 4: backoff + jitter, failed, реестр, replay + +```bash +ops fault arm worker.before_transform transient --count 5 +``` + +```bash +ATT=$(up tests/fixtures/sample.png retry.png); echo "$ATT"; wt $ATT +``` + +```bash +ev $ATT +``` + +> «5 временных ошибок: паузы растут по окнам [1,2], [2,4], [4,8], [8,16] с (видно в `retry_scheduled`). После 5-й попытки — `failed / retry_exhausted`, без бесконечного ретрая». + +```bash +make failed +``` + +```bash +ops replay $ATT --actor koshelev && wt $ATT +``` + +```bash +ops replay $ATT --actor koshelev; echo "exit code $?" +``` + +> «Replay — явная команда оператора с аудитом (событие `replay`, автор). Тот же `job_id` и тот же ключ эффекта. Для уже готового вложения replay **отклоняется** (код 3) — «повторить» не значит исполнить дубль успешной работы». + +Постоянная ошибка — без повторов: + +```bash +python3 -c 'import sys;sys.path.insert(0,"student-tests");from stack import corrupt_png;open("/tmp/broken.png","wb").write(corrupt_png())' +ATT=$(up /tmp/broken.png broken.png); wt $ATT +``` + +```bash +curl -s -o /dev/null -w "attach failed -> %{http_code}\n" -X POST $B/api/v1/chats/$CHAT/messages -H "Authorization: Bearer $TA" -H "Idempotency-Key: demo-$RANDOM" -H 'Content-Type: application/json' -d "{\"text\":\"\",\"attachment_ids\":[\"$ATT\"]}" +``` + +> «Заголовок и CRC корректны, поэтому API принял файл (202). Сломанные сжатые данные выявил только worker → `failed / invalid_image` с первой попытки, без ретраев. Прикрепить failed-вложение — 409». + +## 7. Трек 2: дубли и версии + +```bash +ATT=$(up tests/fixtures/sample.png track2.png); wt $ATT +``` + +```bash +ops republish $ATT --attempt 1; ops republish $ATT --attempt 1; ops republish $ATT --attempt 1; sleep 2; ev $ATT +``` + +> «Три сырых дубля в обход outbox → три `duplicate_noop`. Ни одна строка и ни один объект не изменились». + +```bash +ops reprocess $ATT --pipeline-version 2; sleep 3; st $ATT; ev $ATT +``` + +> «Новая версия pipeline: отдельная job v2 и ключи `v2/…`. Во время перерасчёта вложение оставалось ready со старыми вариантами, переключение — атомарный COMMIT». + +```bash +ops republish $ATT --pipeline-version 1 --attempt 1; sleep 2; ev $ATT | tail -5 +``` + +```bash +ops reprocess $ATT --pipeline-version 1; echo "exit code $?" +``` + +> «Устаревшее v1-сообщение после v2 — no-op; даунгрейд отклонён. Результат старой версии не может перезаписать новый: проверка версии внутри COMMIT и разные ключи S3». + +Сложные гонки (reorder и «v1 медленная, v2 фиксируется первой») — автоматически, ~40 с: + +```bash +python3 student-tests/faults_test.py --only reorder,stale_version_race,poison_message +``` + +Полный набор из 14 сценариев с инвариантами I1–I5 (~5 минут; результат прошлого прогона — в `evidence/faults.txt`): + +```bash +make test-faults +``` + +## 8. Лимиты, persistence, failover, очистка + +```bash +make test-pipeline +``` + +```bash +make test-limits +``` + +```bash +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +```bash +make test-failover ACTION_SCRIPT=scripts/stop-one.sh && ./scripts/restore.sh +``` + +```bash +make sweep-dry-run +``` + +> «Лимиты: при RLIMIT_AS 160 MiB → `image_too_large` за одну попытку; при тайм-ауте 0.3 с дочерний процесс убивается → `retry_exhausted`. Persistence перезапускает API, worker, relay и **все** хранилища, включая RabbitMQ, с теми же volumes. Sweeper в dry-run показывает, что удалил бы, и никогда не трогает queued/processing». + +## 9. Остановка + +```bash +ops fault clear +``` + +```bash +docker compose stop +``` + +> «`stop` сохраняет volumes. Удаление учебных данных — только осознанно: `docker compose down -v`». + +--- ## Ответы на вопросы защиты -**Чем publisher confirm отличается от consumer ack?** Confirm (`basic.ack` от брокера издателю) — брокер принял сообщение и записал его в quorum queue (Raft-лог на диске); после него relay ставит `published_at`. Consumer ack (от worker брокеру) — потребитель закончил обработку и брокер может удалить сообщение; у нас ack отправляется только после COMMIT результата в PostgreSQL. Это две разные границы ответственности: confirm закрывает окно producer → broker, ack — broker → consumer. +**Чем publisher confirm отличается от consumer ack?** Confirm (`basic.ack` от брокера издателю) — брокер принял сообщение и записал его в quorum queue; после него relay ставит `published_at`. Consumer ack (от worker брокеру) — потребитель закончил обработку, брокер может удалить сообщение; у нас ack отправляется только после COMMIT результата в PostgreSQL. Это две разные границы: producer → broker и broker → consumer. -**Что сохранено в момент 202?** Полный оригинал в `staging/` (S3 проверил SHA-256), строка `attachments(queued)` с размером/SHA-256/ключами, `jobs(queued)` и строка `outbox` — всё одной транзакцией. Брокер в этот момент может ещё ничего не знать: публикацию гарантирует outbox. +**Что сохранено в момент 202?** Полный оригинал в `staging/` (S3 проверил SHA-256), строка `attachments(queued)` с размером, SHA-256 и ключами, `jobs(queued)` и строка `outbox` — всё одной транзакцией. Брокер в этот момент может ещё ничего не знать: публикацию гарантирует outbox. -**Почему повторная доставка нормальна и где ключ идемпотентности?** At-least-once неизбежно даёт дубли (падение после confirm до отметки outbox, после COMMIT до ack). Ключ эффекта — `(attachment_id, pipeline_version, variant)`: это и ключи S3, и PK `attachment_variants`; логическая работа — `jobs(job_id)` с UNIQUE `(attachment_id, pipeline_version)`; попытка — `(job_id, attempt)`. Worker стартует попытку только из `queued` с `attempt+1`, всё остальное — no-op с ack. +**Почему повторная доставка нормальна и где ключ идемпотентности?** At-least-once неизбежно даёт дубли (падение после confirm до отметки outbox, после COMMIT до ack). Ключ эффекта — `(attachment_id, pipeline_version, variant)`: это и ключи S3, и PK `attachment_variants`. Логическая работа — `jobs(job_id)` с UNIQUE `(attachment_id, pipeline_version)`, попытка — `(job_id, attempt)`. Worker стартует попытку только из `queued` с `attempt+1`, всё остальное — no-op с ack. -**Как отличить зависший worker от долго работающего?** Живой worker продлевает lease heartbeat'ом каждые 5 с (lease 20 с) — даже при обработке 30 с вторая попытка не стартует. Зависшее *преобразование* убивается по `JOB_TIMEOUT_SECONDS` в дочернем процессе. Умерший worker перестаёт продлевать lease — reaper переводит job в queued. Если «умерший» оживёт, его COMMIT отклонит fencing: job уже не его `(lease_owner, attempt)` → `fenced_out`. Поэтому двух неконтролируемых зафиксированных результатов быть не может. +**Как отличить зависший worker от долго работающего?** Живой worker продлевает аренду heartbeat'ом каждые 5 с (аренда 20 с) — даже при обработке 30 с вторая попытка не стартует. Зависшее преобразование убивается по `JOB_TIMEOUT_SECONDS` в дочернем процессе. Умерший worker перестаёт продлевать аренду — reaper переводит job в queued. Если «умерший» оживёт, его COMMIT отклонит fencing: job уже не его `(lease_owner, attempt)` → `fenced_out`. Поэтому двух неконтролируемых зафиксированных результатов быть не может. diff --git a/REPORT.md b/REPORT.md index 94ad07f6..8595c50f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -9,20 +9,23 @@ Требования: Docker Engine + Compose v2 (проверено: Docker Desktop 28.0.1, Compose 2.33.1), Python 3.9+ для тестов (проверено 3.14, только stdlib), `make`. Стенд: Apple M3, 8 CPU, 16 GiB, Docker VM 8 CPU / 7.6 GiB. +Секреты создаются в `./secrets/` (в `.gitignore`): пароли PG/Valkey/RabbitMQ, S3-ключи, `broker_url`, `rabbitmq.conf`. Для тестов отказов в `.env` поставьте `FAULT_INJECTION=1`. Порядок старта: postgres, valkey, s3, rabbitmq → migrate, s3-init → api ×2, relay, worker, sweeper, ops → lb. + ```sh -./scripts/init-secrets.sh # ./secrets/* (в .gitignore): пароли PG/Valkey/RabbitMQ, S3-ключи, broker_url, rabbitmq.conf -cp .env.example .env # для тестов отказов: FAULT_INJECTION=1 в .env +./scripts/init-secrets.sh +cp .env.example .env docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build --wait -# postgres, valkey, s3, rabbitmq -> migrate, s3-init -> api x2, relay, worker, sweeper, ops -> lb -make check && make test # публичные проверки -python3 student-tests/pipeline_test.py # варианты, EXIF, alpha, лимиты, invalid_image -python3 student-tests/faults_test.py --output evidence/faults.json # 14 сценариев отказов (нужен FAULT_INJECTION=1) -python3 student-tests/limits_test.py # лимиты времени/памяти worker (пересоздаёт worker) +make check && make test +python3 student-tests/pipeline_test.py +python3 student-tests/faults_test.py --output evidence/faults.json +python3 student-tests/limits_test.py make test-persistence ACTION_SCRIPT=scripts/restart.sh -make test-failover ACTION_SCRIPT=scripts/stop-one.sh ; ./scripts/restore.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh && ./scripts/restore.sh ``` +`pipeline_test.py` — варианты, EXIF, alpha, лимиты, `invalid_image`; `faults_test.py` — 14 сценариев отказов (нужен `FAULT_INJECTION=1`); `limits_test.py` — лимиты времени и памяти worker (пересоздаёт контейнер worker). Сценарий защиты по шагам — `DEMO.md`. + - Операторские действия: `./scripts/stop-worker.sh`, `./scripts/start-worker.sh`, CLI `docker compose exec ops python -m app.ops `. RabbitMQ management UI — только `http://127.0.0.1:15672` (логин `messenger`, пароль в `secrets/rabbitmq_password`). - Bootstrap идемпотентен: `migrate` применяет только новые файлы под advisory lock; `s3-init` — «bucket already exists»; топологию брокера (exchange/queues/bindings) объявляют relay/worker/ops при каждом старте (`declare` идемпотентен). - Безопасная остановка: `docker compose stop` / `docker compose down` (volumes сохраняются). **Намеренное удаление учебных данных:** `docker compose down -v` (удаляет `pgdata`, `s3data`, `rabbitmqdata`). -- 2.54.0