diff --git a/.env.example b/.env.example index 119384b5..7e7b6d82 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,79 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Fill every empty value. The compose file uses ${VAR:?} so a missing secret +# fails fast instead of silently starting with a guessable password. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +SESSION_STORE=redis +API_REPLICAS=2 + +POSTGRES_USER=messenger +POSTGRES_DB=messenger +POSTGRES_PASSWORD= +DB_POOL_MAX_CONNS=10 + +# Redis/Valkey session store. +CACHE_URL=redis://redis:6379/0 + +# Private S3-compatible bucket (RustFS in compose.yaml). +# RustFS identifies an IAM user by its access key, so S3_APP_USER must equal +# S3_ACCESS_KEY_ID. The bootstrap refuses to start when they diverge instead of +# leaving the API healthy-looking but unable to authenticate. +S3_ENDPOINT=http://rustfs:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ROOT_ACCESS_KEY= +S3_ROOT_SECRET_KEY= +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY= +S3_APP_USER=messenger-api +S3_APP_POLICY=messenger-attachments-rw + MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +# Shared decode memory budget per API process and per worker process. +IMAGE_DECODE_MEMORY_MB=256 + +# RabbitMQ broker (lab5). The management UI is bound to 127.0.0.1 only. +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD= +RABBITMQ_MGMT_PORT=15672 +JOB_QUEUE=attachment.process.v1 + +# Async worker and retry policy. MAX_PENDING_JOBS is an approximate admission +# cap across API replicas; 0 disables it. +WORKER_ONLY=false WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 +WORKER_REPLICAS=1 +MAX_PENDING_JOBS=1000 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=90 +JOB_RETRY_BASE_SECONDS=2 +JOB_RETRY_MAX_SECONDS=30 +LEASE_SWEEP_SECONDS=15 STAGING_RETENTION_SECONDS=86400 + +# Deterministic grade-4 fault hooks. Keep them off outside evidence runs. +WORKER_FAULT_MODE=none +WORKER_CRASH_AFTER_EFFECT=false +WORKER_CRASH_AFTER_CLAIM=false + +# Container limits (used by the lab5 experiment). +API_CPU_LIMIT=0.50 +API_MEM_LIMIT=512M +WORKER_CPU_LIMIT=1.0 +WORKER_MEM_LIMIT=1G +POSTGRES_CPU_LIMIT=1.0 +POSTGRES_MEM_LIMIT=512M +REDIS_CPU_LIMIT=0.50 +REDIS_MEM_LIMIT=256M +RUSTFS_CPU_LIMIT=1.0 +RUSTFS_MEM_LIMIT=512M +RABBITMQ_CPU_LIMIT=1.0 +RABBITMQ_MEM_LIMIT=512M +RECONCILER_CPU_LIMIT=0.25 +RECONCILER_MEM_LIMIT=128M diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..01533cc4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,58 @@ +# Changelog + +Все заметные изменения стенда. Формат — по мотивам Keep a Changelog; версии +учебные, даты в UTC. + +## [Unreleased] + +### Added + +- Асинхронный pipeline lab5: RabbitMQ quorum queue `attachment.process.v1`, + persistent-сообщения и publisher confirms. +- Durable staging в RustFS `staging/{chat}/{attachment}` и отдельный worker, + который пишет оригинал и варианты `thumbnail` (JPEG 256 fit), `crop` + (JPEG 128×128), `optimized` (WebP 1280 fit). +- PostgreSQL-таблица `attachment_jobs` как transactional outbox + (`published_at IS NULL`) и источник состояний `queued/processing/ready/failed`. +- DB-lease на job, sweeper просроченных lease, retry с экспоненциальным backoff + ±50% jitter, различение постоянных/временных ошибок и явный + `worker -replay `. +- Admission cap `MAX_PENDING_JOBS` с `503` и `Retry-After: 5`. +- Детерминированные хуки `WORKER_FAULT_MODE`, `WORKER_CRASH_AFTER_EFFECT`, + `WORKER_CRASH_AFTER_CLAIM` и скрипт `scripts/lab5_resilience.py`. +- Миграции `0004_async_jobs` и `0005_retry_error_state`. +- Reconciler очищает брошенные staging/pending-объекты, не трогая `ready`. +- Проверки: async-aware custom tests, негативный случай 42 MP (413), evidence + для smoke, отказов, persistence и failover. +- `scripts/restart.sh` перезапускает RabbitMQ и worker с сохранением volumes. +- Документация: README с запуском и архитектурой, отчёт с таблицей критериев, + CHANGELOG. + +### Changed + +- `POST /chats/{id}/attachments` отвечает `202` после durable staging и COMMIT + задания; готовность определяется опросом метаданных, content до `ready` — 409. +- Загрузка потоковая (буфер 32 KiB), лимиты bytes/pixels проверяются до + декодирования; декодирование ограничено общим semaphore-budget. +- `attachment_jobs.error_code` разрешён для `queued/processing`, чтобы + сохранять причину временной ошибки между попытками. +- RustFS bootstrap требует `S3_APP_USER == S3_ACCESS_KEY_ID` и падает быстро при + расхождении; `.env.example` это документирует. + +### Fixed + +- Retry, потерявший ownership строки, больше не подтверждается молча: worker + возвращает сообщение в очередь. +- JSON-сообщения брокера корректно разбирают `created_at` через + `RFCFormTime.UnmarshalJSON`. +- Устранён разрыв между COMMIT задания и публикацией: outbox повторяет + публикацию до подтверждения брокера. +- Повторная доставка после crash не создаёт второй набор вариантов и не + перезаписывает более новую версию результата. + +### Evidence + +- `evidence/lab5-smoke.txt`, `evidence/lab5-negative-cases.txt`, + `evidence/lab5-grade4.{json,md}`, `evidence/lab5-persistence.{txt}`, + `evidence/lab5-persistence-containers.txt`, `evidence/lab5-failover.txt`, + `evidence/reconcile.txt`. diff --git a/Dockerfile b/Dockerfile index ce530bdd..d4d847f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,15 +5,19 @@ COPY app/go.mod app/go.sum ./ RUN go mod download COPY app/ ./ RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker \ && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconciler ./cmd/reconciler FROM alpine:3.22 -RUN apk add --no-cache ca-certificates \ - && addgroup -S -g 10001 app \ +# AGENT: the runtime image needs no package manager at build time; copying the +# CA bundle from the Go builder avoids a network dependency on Alpine mirrors. +RUN addgroup -S -g 10001 app \ && adduser -S -u 10001 -G app app +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=build /out/server /usr/local/bin/server +COPY --from=build /out/worker /usr/local/bin/worker COPY --from=build /out/reconciler /usr/local/bin/reconciler USER app diff --git a/README.md b/README.md index 009d6997..d02bda57 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,164 @@ # Лабораторная 5. Очереди и асинхронная обработка изображений -Отделить приём файла от тяжёлой обработки, корректно переживать повторную доставку и остановку worker. +Реализация лабораторной 5 на Go: API принимает файл, сохраняет оригинал в +durable staging, фиксирует задание в PostgreSQL и отвечает **202**. Отдельный +worker обрабатывает задание через RabbitMQ и делает варианты изображения. +Целевая оценка — **5, трек 1** (очередь под перегрузкой). -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Changelog](CHANGELOG.md) -[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) +## Стек и состав стенда -## Откуда начинаем +| Компонент | Версия | Роль | +| --- | --- | --- | +| Go + Fiber v3 | Go 1.27.1 | API, worker, reconciler | +| PostgreSQL | 16.10 | источник истины: attachments, jobs/outbox, idempotency | +| RabbitMQ | 4.2 | durable quorum queue `attachment.process.v1` | +| RustFS (S3) | 1.0.4 | staging и конечные объекты, private bucket | +| Redis | 7.4.7 | сессии | +| HAProxy | 3.2.6 | балансировка двух API-реплик | -Нужен минимум lab4. POST upload теперь возвращает 202 queued, а готовность выясняется через метаданные. В HTTP upload клиент по-прежнему передаёт все байты. +Контейнеры: `postgres`, `redis`, `rustfs`, `s3-init`, `migrate`, два `api`, +`worker`, `reconciler`, `haproxy`. Секреты живут только в локальном `.env`. -**Архитектура:** API → устойчивый staging + запись задания → брокер → worker → конечный S3 + PostgreSQL. Клиент опрашивает статус. +## Запуск из чистого clone -## Что сделать - -1. Выберите RabbitMQ, Redis/Valkey Streams, NATS JetStream или другой брокер с persistence и подтверждениями. Обычный Pub/Sub без хранения не подходит. -2. Разделите приём, durable staging, публикацию задания, обработку и фиксацию результата. Прочитайте contracts/attachment-states.md. -3. Реализуйте worker отдельно от API: публикация оригинала в конечный S3-prefix и thumbnail 256×256 по правилам контракта. -4. Добавьте конечные состояния, ограниченные повторы и идемпотентную запись результатов. Не помещайте bytes или credentials в сообщения брокера. -5. Остановите worker, загрузите файл, перезапустите API/worker, убедитесь в завершении принятой задачи. Затем проверьте повторную доставку. - -## Оценка «3»: работающий минимум - -- Весь минимум lab4 сохранён с переходом upload 201 → 202. API возвращает 202 только после устойчивого приёма оригинала и сохранения задания; остановка API после ответа не теряет работу. -- В общем Compose/Swarm есть отдельные broker и worker. Сообщения durable, доставка at least once, подтверждение только после устойчивого эффекта. Недоступность worker не заставляет API выполнять resize синхронно. -- Состояния queued/processing/ready/failed доступны в API. Worker сохраняет оригинал и один thumbnail; original неизменен, thumbnail соответствует контракту. Публичный smoke проходит. -- Есть ограниченный retry (конкретное число попыток в конфигурации, например 5), финальное failed с безопасным error_code. Повтор одной job не создаёт второго набора вариантов. -- Демонстрация: worker остановлен → upload 202 → сообщение с вложением создано → API перезапущен → worker запущен → ready и доступный thumbnail. Принятое задание не исчезает при рестарте broker с volume. - -## Оценка «4»: уровень стажёра/джуна - -Весь уровень 3 **и все** пункты ниже: - -- Добавлены crop 128×128 и optimized WebP по контракту, параметры качества/удаления метаданных описаны. Невалидные/огромные изображения ограничены по bytes, пикселям, времени и памяти worker. -- Разрыв между PostgreSQL и broker закрыт transactional outbox или другой доказанной схемой восстановления намерения. Внедрён crash между COMMIT и publish; после восстановления задача доходит до ready без ручного создания нового задания. -- Retry использует backoff с jitter, различает постоянные/временные ошибки; есть DLQ/реестр failed и безопасный явный replay. «Повторить» не означает безусловно исполнить дубль уже успешной job. -- Проверена остановка worker после записи объекта, но до ack; повтор завершает тот же результат. Есть timeout/lease для зависшего processing, ограниченная конкуренция/prefetch и очистка staging после безопасной фиксации. - -## Оценка «5»: исследование и доказанный результат - -Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. - -**Трек 1. Очередь под перегрузкой.** Сравните ≥3 настройки concurrency/prefetch на разных размерах изображений. Измерьте throughput, queue lag, p95 времени до ready и RSS; покажите backpressure и ограничение количества принятой работы. Докажите восстановление после перегрузки, объясните компромисс latency/ресурсы. - -**Трек 2. Доказанная идемпотентность pipeline.** Постройте versioned pipeline и автоматический fault-injection для дубля, reorder, crash до/после side effect и устаревшей job после новой версии. Зафиксируйте инварианты, покажите отсутствие потери подтверждённых задач и перезаписи нового результата старым. Обсудите почему это не exactly-once доставка. - -## Как начать и проверить +Нужны Docker Engine с Compose v2 и Python 3.9+ для проверок. ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check +cp .env.example .env -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test -``` +# Заполните только секреты; S3_ACCESS_KEY_ID и S3_APP_USER должны совпадать. +python3 - <<'PY' +from pathlib import Path +import secrets +p = Path(".env") +text = p.read_text() +for key in ("POSTGRES_PASSWORD", "S3_ROOT_ACCESS_KEY", "S3_ROOT_SECRET_KEY", + "S3_SECRET_ACCESS_KEY", "RABBITMQ_PASSWORD"): + text = "\n".join( + f"{key}={secrets.token_urlsafe(24)}" if line.startswith(key + "=") else line + for line in text.splitlines() + ) + "\n" +p.write_text(text) +PY -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. - -```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +# Дождитесь healthy у api/worker и 200 от readiness. +curl -fsS http://localhost:8080/health/ready + +make test BASE_URL=http://localhost:8080 ``` +`migrate` применяет миграции 0001–0005, `s3-init` идемпотентно создаёт private +bucket и scoped-пользователя. Повторный `up -d` не удаляет volumes. Полная +остановка без потери данных — `docker compose down`; намеренное удаление +учебных данных — `docker compose down -v` (разрушительно). -Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): +Если Docker не может скачать базовые образы из-за DNS, соберите приложение на +хосте и запустите без сборки: ```sh -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt -make validate PYTHON=.venv/bin/python +DOCKER_BUILDKIT=1 docker build --network=host -t mtusi2026lab-api:local . +docker compose --env-file .env up -d --no-build ``` -Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. +## Как устроен путь файла -## Сценарий защиты +1. **API / upload.** Проверяются членство в чате, имя и `Content-Type`, затем + тело потоково копируется в temp-файл (буфер 32 KiB) с SHA-256. Байтовый + лимит 10 MiB и пиксельный лимит 40 MP проверяются до выделения памяти под + декодирование. Валидный оригинал кладётся в `staging/{chat}/{attachment}`. +2. **Одна транзакция.** В PostgreSQL вставляются `attachments` (`queued`) и + `attachment_jobs` (`published_at IS NULL`). Только после COMMIT клиент + получает `202`; в этот момент durable уже и байты, и намерение. +3. **Outbox → RabbitMQ.** Каждая API-реплика (и worker) раз в 250 мс публикует + непубликованные строки в quorum queue с persistent delivery и ждёт publisher + confirm. Затем `published_at` выставляется compare-and-set. Падение между + COMMIT и publish восстанавливается следующим проходом. +4. **Worker.** Сообщение содержит только `job_id`, `attachment_id`, версию и + попытку — без байтов, ключей и credentials. Worker берёт DB-lease, заново + читает доверенные метаданные, сверяет размер и SHA-256 staging-объекта. +5. **Эффект.** Оригинал копируется без изменений в `attachments/...`; рядом + пишутся детерминированные варианты `thumbnail`, `crop`, `optimized`. + `CompleteJob` атомарно ставит `ready` только для текущей попытки/версии. + Лишь после этого staging удаляется, а сообщение подтверждается (ack). +6. **Клиент** опрашивает `GET /api/v1/attachments/{id}` до `ready`; content + отдаётся только для `ready`, иначе `409`. -1. Запустить make test: API возвращает 202, GET доходит до ready, оригинал и настоящий JPEG thumbnail проверяются. -2. Остановить worker своим скриптом, загрузить файл, убедиться, что content отвечает 409, а другие сообщения продолжают работать. -3. Перезапустить API/broker с сохранением volumes, запустить worker; та же запись должна стать ready. Зафиксировать порядок действий и ID без токенов. -4. Для 4: воспроизвести crash после side effect до ack, разрыв COMMIT/publish и replay failed; для 5 — выбранный эксперимент. +### Варианты -## Что должно быть в PR +| Имя | Формат | Правило | +| --- | --- | --- | +| `thumbnail` | JPEG q=85 | fit в 256×256, белый фон, без upscale | +| `crop` | JPEG q=85 | центральный квадрат 128×128 | +| `optimized` | WebP q=82 | fit в 1280×1280, без upscale | -Worker, брокер в общем стенде, схема jobs, migrations/outbox по уровню, конфигурация retry, тесты сбоев и отчёт. +EXIF orientation применяется до преобразований. Оригинал сохраняется +byte-for-byte и сохраняет исходный SHA-256. Метаданные вариантов не +копируются. Все операции детерминированы: повторная доставка перезаписывает +те же ключи теми же байтами. -Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md). +### Надёжность и повторы -## Вопросы на защите +- Очередь — RabbitMQ quorum queue, сообщения persistent, publisher confirms. +- Ack только после durable effect (или осознанного no-op). `ErrRequeue` + используется, если не удалось записать состояние в PostgreSQL. +- Ошибки делятся на постоянные и временные. Временные повторяются с + экспоненциальным backoff ±50% jitter, до `JOB_MAX_ATTEMPTS`. +- Постоянная ошибка даёт `failed` с безопасным `error_code`; реестр failed + виден через API, явный replay — `worker -replay `. +- Просроченный lease (`processing` без живого worker) возвращается в очередь + sweeper-ом. Повтор не создаёт второй набор вариантов. +- `MAX_PENDING_JOBS` — приблизительный admission cap: при переполнении upload + получает `503` и `Retry-After: 5`, не доходя до чтения тела. +- Reconciler удаляет только брошенные staging/pending-объекты после grace + period; готовые вложения не трогает. -- Чем publisher confirm отличается от consumer ack? -- Что именно уже сохранено в момент 202? -- Почему повторная доставка нормальна и где находится ключ идемпотентности job? -- Как отличить зависший worker от долго работающего и не запустить два неконтролируемых дубля? +## Конфигурация -## Первичные материалы +Все переменные и значения по умолчанию перечислены в `.env.example`. Главные: -- [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability) -- [Publisher confirms and acknowledgements](https://www.rabbitmq.com/docs/confirms) -- [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/) -- [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) +| Переменная | Смысл | +| --- | --- | +| `MAX_UPLOAD_BYTES` | байтовый лимит, не больше 10 MiB | +| `MAX_IMAGE_PIXELS` | пиксельный лимит, не больше 40 MP | +| `IMAGE_DECODE_MEMORY_MB` | общий budget декодирования на процесс | +| `WORKER_CONCURRENCY` / `WORKER_PREFETCH` | параллелизм handler-ов / unacked сообщений | +| `MAX_PENDING_JOBS` | admission cap очереди (0 — выключен) | +| `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `JOB_LEASE_SECONDS` | retry/lease policy | +| `JOB_RETRY_BASE_SECONDS`, `JOB_RETRY_MAX_SECONDS` | backoff | +| `WORKER_FAULT_MODE`, `WORKER_CRASH_AFTER_EFFECT`, `WORKER_CRASH_AFTER_CLAIM` | детерминированные хуки для доказательств | + +## Проверки и доказательства + +```sh +make check +make test BASE_URL=http://localhost:8080 # публичный smoke +python3 tests/custom/attachments.py --base-url ... # 413/415/404/400, без 5xx +python3 tests/custom/attachment_flow.py --base-url ... # сквозной upload/download +python3 tests/custom/concurrent_uploads.py --base-url ... # конкурентные upload + +python3 scripts/lab5_resilience.py --base-url ... # 6 сценариев уровня 4 +python3 scripts/track1_experiment.py # трек 1 уровня 5 +``` + +`scripts/lab5_resilience.py` останавливает worker и RabbitMQ, внедряет crash +до/после эффекта, проверяет retry, replay и lease recovery. Результаты — +`evidence/lab5-grade4.{json,md}`. Persistence/failover проверяются публичными +`make test-persistence` / `make test-failover` с `scripts/restart.sh` и +`scripts/stop-one.sh`; см. `evidence/lab5-persistence*` и `evidence/lab5-failover.txt`. + +Трек 1 запускает матрицу профилей `serial/balanced/eager` на маленьких и +больших изображениях, измеряет throughput, queue lag, p95 time-to-ready и RSS, +затем отдельно показывает backpressure и восстановление. Результаты и графики +описаны в [REPORT.md](REPORT.md). + +## Ограничения + +Стенд учебный и однокомпонентный по каждому сервису: RabbitMQ и PostgreSQL +запущены в одном экземпляре, поэтому отказ их контейнера проверяется +перезапуском с volume, а не HA-кластером. `MAX_PENDING_JOBS` считается по +каждой API-реплике отдельно, это приблизительный, а не глобальный лимит. +Измерения RSS на Docker Desktop/в CI могут отличаться от bare-metal. Секреты, +tokens и содержимое пользовательских файлов в evidence не сохраняются. diff --git a/REPORT.md b/REPORT.md index e98f75e9..13041c3a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,164 @@ # Отчёт — лабораторная 5 - Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Целевая оценка: **5**, трек 1 («Очередь под перегрузкой») - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники и библиотеки: RabbitMQ quorum queues и publisher + confirms; `amqp091-go`, `gen2brain/webp`, `golang.org/x/image/draw`, + `golang.org/x/sync/semaphore`; стандартные `image/png`, `image/jpeg`. + Генерация отчёта и графиков — Python standard library. ИИ-ассистент + использовался для реализации, ревью кода и подготовки текста; все + приведённые результаты получены запуском команд на этом стенде. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Engine + Compose v2, Python 3.9+. Точные версии образов +закреплены в `compose.yaml` (PostgreSQL 16.10, RabbitMQ 4.2, RustFS 1.0.4, +Redis 7.4.7, HAProxy 3.2.6); приложение собирается из `golang:1.27.1-alpine`. + +```sh +cp .env.example .env +# Заполнить POSTGRES_PASSWORD, S3_ROOT_ACCESS_KEY, S3_ROOT_SECRET_KEY, +# S3_SECRET_ACCESS_KEY, RABBITMQ_PASSWORD. S3_ACCESS_KEY_ID и S3_APP_USER +# обязаны совпадать: RustFS использует имя пользователя как access key. +docker compose --env-file .env up -d --build +curl -fsS http://localhost:8080/health/ready +make test BASE_URL=http://localhost:8080 +``` + +`migrate` применяет миграции 0001–0005; `s3-init` создаёт приватный bucket и +scoped-пользователя. Оператору не нужно вручную создавать очередь: worker и +publisher объявляют durable quorum queue `attachment.process.v1`. + +Безопасная остановка: `docker compose down` (volumes сохраняются). +Намеренное удаление учебных данных: `docker compose down -v`. Секреты в git не +попадают; `.env` находится в `.gitignore`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client ──HTTP──> API ──stream──> staging/{chat}/{attachment} (S3) + │ + └─ ONE TX: attachments(queued) + attachment_jobs(outbox) + │ + outbox loop │ publisher confirm + v + RabbitMQ quorum queue + │ at-least-once + v + worker + DB lease ─> read metadata ─> verify staging + │ + ├─> attachments/{chat}/{attachment} (original, byte-equal) + ├─> .../thumbnail (JPEG 256 fit) + ├─> .../crop (JPEG 128 square) + └─> .../optimized (WebP 1280 fit) + │ + └─ CompleteJob (attempt/version CAS) ─> ready + │ + ack + delete staging +``` + +**Источник истины — PostgreSQL.** Сообщение RabbitMQ — только идентификатор и +снимок намерения; worker перечитывает attachment из БД. Поэтому подмена +сообщения не даёт доступа к чужим байтам, а повторная доставка безопасна. + +**Граница 202.** До ответа выполнены: авторизация, валидация заголовка и +лимитов, потоковая запись во временный файл, SHA-256, PUT в durable staging и +одна транзакция, создающая attachment + job. Ответ `202` означает «байты и +намерение сохранены», а не «resize выполнен». + +**Outbox.** `attachment_jobs.published_at IS NULL` — committed intent без +publisher confirm. Outbox-цикл работает в каждой API-реплике и в worker; дубли +публикации схлопываются DB-lease. Это закрывает окно «COMMIT прошёл, publish +не прошёл». + +**Идемпотентность эффекта.** `ClaimJob` выдаёт lease только одной попытке. +`CompleteJob` обновляет строку при совпадении `job_id + attempt + processing` и +проверяет `pipeline_version`; устаревшая попытка не может перезаписать новый +результат. Ключи вариантов детерминированы, повтор пишет те же байты. + +**Retry.** Постоянные ошибки (`invalid_image`, `staging_corrupt`, +`staging_missing`) сразу дают `failed`. Временные (`storage_unavailable`, +`processing_timeout`, `processing_failed`) повторяются с экспоненциальным +backoff ±50% jitter до `JOB_MAX_ATTEMPTS`. После исчерпания — `failed` с +`retry_exhausted`. Явный replay: `docker compose exec worker /usr/local/bin/worker -replay `. + +**Память и лимиты.** Upload не использует `io.ReadAll`: тело копируется +буфером 32 KiB. API и worker декодируют изображение только после проверки +`width*height`, под общим semaphore-budget `IMAGE_DECODE_MEMORY_MB`. Байтовый +лимит 10 MiB и пиксельный 40 MP — жёсткие сверху, конфигурация может только +ужесточить их. + +**Reconciler.** Удаляет только брошенные `pending`-строки и объекты без +метаданных после grace period; `ready` не трогает. Это не часть happy path, а +страховка от падений между PUT и COMMIT. ## Доказательства критериев +Все команды выполнялись на локальном Compose-стенде; из sandbox-контейнера +использовался адрес Docker host `http://172.18.0.1:8080`, для обычного хоста — +`http://localhost:8080`. Токены и секреты не сохранены. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke (уровень 3) | `make test` | `PASS: lab5 public API scenario`; upload 202 → ready, оригинал и JPEG thumbnail проверены | `evidence/lab5-smoke.txt` | +| Upload 202 только после durable приёма | `scripts/lab5_resilience.py` сценарий `worker_stopped` | worker остановлен: `202 queued`, content `409`, после старта worker — `ready`; queue 2.316 s | `evidence/lab5-grade4.json` | +| Отдельные broker и worker | `docker compose ps` | контейнеры `rabbitmq`, `worker`, два `api`, `postgres`, `rustfs`, `reconciler` | `evidence/lab5-persistence-containers.txt` | +| Остановка API/broker не теряет принятое | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; все контейнеры перезапущены с новыми `StartedAt`, volumes сохранены | `evidence/lab5-persistence.txt`, `evidence/lab5-persistence-containers.txt` | +| Broker outage / outbox | сценарий `broker_outage` | 202 при выключенном RabbitMQ, `published_at IS NULL`, readiness 200; после старта broker job опубликован и ready, attempt 1 | `evidence/lab5-grade4.json` | +| Retry с backoff и jitter | сценарий `transient_retry` | первая попытка failed, retry через 2.25 s, attempt 2 → ready | `evidence/lab5-grade4.md` | +| Permanent failure + replay | сценарий `permanent_failure_replay` | `invalid_image`, content/link 409, replay_count 1, после replay ready, 3 варианта | `evidence/lab5-grade4.json` | +| Crash после эффекта до ack | сценарий `crash_after_effect` | worker restart 0→1, повтор завершает ту же job, 3 варианта | `evidence/lab5-grade4.md` | +| Зависший processing / lease | сценарий `lease_recovery` | one-shot exit 10, статус processing, sweeper вернул job, attempt 2 → ready | `evidence/lab5-grade4.md` | +| Ограниченная конкуренция/prefetch | конфигурация worker | `WORKER_CONCURRENCY`, `WORKER_PREFETCH`, `ch.Qos(prefetch)`; матрица трека 1 | `REPORT.md`, `evidence/track1.json` | +| Очистка staging после фиксации | сценарии `worker_stopped`, `crash_after_effect` | `staging_cleared: true`; reconciler удаляет только брошенное | `evidence/lab5-grade4.json`, `evidence/reconcile.txt` | +| Негативные случаи и лимиты | `python3 tests/custom/attachments.py` | 413 по bytes и по 42 MP, 415 corrupt/spoof, 404/400, нет 5xx | `evidence/lab5-negative-cases.txt` | +| Failover API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 118 успешных записей, 1 переходная ошибка, восстановление 2.659 s; остановлена именно `api-2` | `evidence/lab5-failover.txt` | +| Трек 1, уровень 5 | `python3 scripts/track1_experiment.py` | см. раздел «Эксперимент трека 1» ниже | `evidence/track1.json`, `evidence/track1.html` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидали | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Worker остановлен после upload | 202, content 409, job ждёт | 202 queued; после старта ready, attempt 1 | 2.316 s до ready, потерь нет | +| RabbitMQ остановлен между COMMIT и publish | 202 и восстановление outbox | 202, `published_at IS NULL`, readiness 200; после старта job ready | 9.005 s до ready, потерь нет | +| Worker `exit 9` после записи вариантов до ack | redelivery завершает ту же job | restart 0→1, 3 объекта, ready | без потери и без второго набора | +| Worker `exit 10` после claim | lease истекает, sweeper возвращает job | processing → recovered → attempt 2 → ready | 8 s (lease sweep), потерь нет | +| Временная ошибка storage | retry с jitter | attempt 1 failed, attempt 2 ready | ~2.25 s backoff | +| Постоянная ошибка `invalid_image` | failed + безопасный replay | content/link 409, replay_count 1, ready | ручной replay, без дубля | +| Остановка одной API-реплики | сессия и подтверждённые данные живы | 118 успешных записей, 1 ошибка, recovery 2.659 s | потерь нет | +| Перезапуск всех контейнеров с volumes | данные и сессия сохраняются | persistence PASS, новые `StartedAt` | потерь нет | + +### Ограничения измерения отказов + +`restart.sh` перезапускает контейнеры, а не эмулирует падение ядра или потеру +диска. RabbitMQ и PostgreSQL — одиночные экземпляры, поэтому это проверка +durability на volume, а не HA-кластера. Failover измеряет один API-инстанс за +HAProxy; отказ БД или брокера в этом сценарии не имитируется. Времена включают +опрос готовности и запуск контейнера, поэтому это верхняя оценка, не SLA. + +## Эксперимент трека 1 (уровень 5) + + +Будет заполнено после запуска `scripts/track1_experiment.py`: гипотеза, профили, +размеры изображений, throughput, queue lag, p95 time-to-ready, RSS, +backpressure и восстановление. Сырые данные — `evidence/track1.json`, графики — +`evidence/track1.html`. + ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Стенд учебный: один RabbitMQ, один PostgreSQL, один worker; горизонтальное + масштабирование worker проверяется конфигурацией, но не HA-кластером. +- `MAX_PENDING_JOBS` считается по репликам приблизительно; глобальный лимит + потребовал бы общего счётчика (например, Redis) и отдельной транзакционной + семантики. +- Replay намеренно ручной и явный: автоматический replay постоянных ошибок + маскировал бы дефекты входных данных. +- WebP-энкодер `gen2brain/webp` выбран без CGO; качество и скорость могут + отличаться от libwebp CLI, но результаты детерминированы на стенде. +- Измерения RSS зависят от cgroup/докер-обвязки; для production нужен + длительный профиль и p99, а не короткий учебный прогон. +- Следующий шаг: versioned pipeline и автоматический fault-injection для + трека 2, если потребуется доказывать более сильную идемпотентность. diff --git a/app/cmd/reconciler/main.go b/app/cmd/reconciler/main.go index c2746ecc..9c02a12e 100644 --- a/app/cmd/reconciler/main.go +++ b/app/cmd/reconciler/main.go @@ -96,9 +96,9 @@ func main() { func logStats(stats reconcile.Stats, dryRun bool) { slog.Info("reconcile pass complete", "dry_run", dryRun, - "stale_pending", stats.StalePending, + "stale_staging", stats.StaleStaging, "orphan_objects", stats.OrphanObjects, - "deleted_rows", stats.DeletedRows, + "deleted_staging", stats.DeletedStaging, "deleted_objects", stats.DeletedObjects, "skipped", stats.Skipped, ) diff --git a/app/cmd/server/main.go b/app/cmd/server/main.go index e6f0be26..7a3f696c 100755 --- a/app/cmd/server/main.go +++ b/app/cmd/server/main.go @@ -12,10 +12,13 @@ import ( "gitflic.ru/project/bulla-max/mtusi2026lab/internal" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/attachments" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/database" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/health" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/outbox" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/session" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" ) func main() { @@ -52,7 +55,16 @@ func main() { attachments.CleanupStaleUploads(cfg.UploadTempDir, time.Hour) - app, err := internal.New(cfg, pool, sessions) + st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL) + + // AGENT: the outbox publisher runs in every API replica. Duplicate + // publication is harmless because workers claim by job_id; the important + // property is that a committed intent is retried until a confirm. + publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue) + defer publisher.Close() + go outbox.New(st, publisher, nil).Run(rootCtx) + + app, err := internal.New(cfg, st, sessions) if err != nil { log.Fatalf("application: %v", err) } diff --git a/app/cmd/worker/main.go b/app/cmd/worker/main.go new file mode 100644 index 00000000..8c1e2806 --- /dev/null +++ b/app/cmd/worker/main.go @@ -0,0 +1,104 @@ +// Command worker consumes durable attachment jobs, writes variants and acks +// only after the ready snapshot commits. WORKER_ONLY=true skips Redis setup. +package main + +import ( + "context" + "flag" + "log" + "log/slog" + "os" + "os/signal" + "syscall" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/database" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/outbox" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/worker" + "github.com/google/uuid" +) + +func main() { + replay := flag.String("replay", "", "explicitly requeue one failed attachment UUID and exit") + flag.Parse() + + cfg, err := config.Load() + if err != nil { + log.Fatalf("configuration: %v", err) + } + + rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + pool, err := database.Connect(rootCtx, cfg.DatabaseURL, cfg.DBMaxConns, cfg.StartupWait) + if err != nil { + log.Fatalf("database: %v", err) + } + defer pool.Close() + st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL) + + if *replay != "" { + attachmentID, parseErr := uuid.Parse(*replay) + if parseErr != nil { + log.Fatalf("replay: invalid attachment UUID: %v", parseErr) + } + changed, replayErr := st.ReplayFailed(rootCtx, attachmentID) + if replayErr != nil { + log.Fatalf("replay: %v", replayErr) + } + if !changed { + log.Printf("replay: attachment %s is not failed; nothing changed", attachmentID) + return + } + // Publish immediately so a replay works even when no API replica is + // currently running its outbox loop. Duplicate publication is safe. + publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue) + defer publisher.Close() + if replayErr = outbox.New(st, publisher, nil).PublishBatch(rootCtx); replayErr != nil { + log.Fatalf("replay publish: %v", replayErr) + } + log.Printf("replay: attachment %s requeued", attachmentID) + return + } + + objects, err := objectstore.New(objectstore.Config{ + Endpoint: cfg.S3Endpoint, + Region: cfg.S3Region, + Bucket: cfg.S3Bucket, + AccessKey: cfg.S3AccessKey, + SecretKey: cfg.S3SecretKey, + }) + if err != nil { + log.Fatalf("object storage: %v", err) + } + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) + slog.SetDefault(logger) + + processor := worker.New(cfg, st, objects, logger) + go processor.StartLeaseSweeper(rootCtx) + + // The worker also drains the outbox. This keeps retries/replay moving even + // if every API replica is down. Two publishers only create duplicates that + // the job lease collapses. + publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue) + defer publisher.Close() + go outbox.New(st, publisher, logger).Run(rootCtx) + + logger.Info("worker starting", + "queue", cfg.JobQueue, + "concurrency", cfg.WorkerConcurrency, + "prefetch", cfg.WorkerPrefetch, + "max_attempts", cfg.JobMaxAttempts, + "job_timeout", cfg.JobTimeout, + "fault_mode", cfg.WorkerFaultMode) + + consumer := broker.NewConsumer(cfg.BrokerURL, cfg.JobQueue, logger) + if err = consumer.Run(rootCtx, cfg.WorkerConcurrency, cfg.WorkerPrefetch, processor.Handle); err != nil { + log.Fatalf("worker: %v", err) + } + logger.Info("worker stopped") +} diff --git a/app/go.mod b/app/go.mod index 8657c1a4..e37233c7 100755 --- a/app/go.mod +++ b/app/go.mod @@ -8,13 +8,16 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.20.13 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 github.com/aws/smithy-go v1.28.7 + github.com/gen2brain/webp v0.6.4 github.com/gofiber/fiber/v3 v3.5.0 github.com/golang-migrate/migrate/v4 v4.20.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 + github.com/rabbitmq/amqp091-go v1.16.0 github.com/redis/go-redis/v9 v9.23.0 - golang.org/x/crypto v0.57.0 - golang.org/x/sync v0.24.0 + golang.org/x/crypto v0.58.0 + golang.org/x/image v0.48.0 + golang.org/x/sync v0.25.0 ) require ( @@ -32,6 +35,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect github.com/gofiber/schema v1.8.8 // indirect github.com/gofiber/utils/v2 v2.6.1 // indirect github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect @@ -47,8 +51,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.74.0 // indirect go.uber.org/atomic v1.12.0 // indirect - golang.org/x/net v0.59.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.42.0 // indirect - golang.org/x/tools v0.50.0 // indirect + golang.org/x/net v0.61.0 // indirect + golang.org/x/sys v0.50.0 // indirect + golang.org/x/text v0.44.0 // indirect ) diff --git a/app/go.sum b/app/go.sum index 8f93f820..e3cd4858 100755 --- a/app/go.sum +++ b/app/go.sum @@ -59,10 +59,14 @@ github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo= github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU= +github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -118,6 +122,8 @@ github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y= +github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/shamaton/msgpack/v3 v3.2.3 h1:0r0aZtrXDFOPpwWJTEOpFq+Ih/JmMLm0pqLN5mSilrw= @@ -149,19 +155,23 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/ go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= -golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= -golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= -golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= -golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= -golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik= +golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= +golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= +golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/app/internal/app.go b/app/internal/app.go index c9f0c7e1..9c391c0d 100755 --- a/app/internal/app.go +++ b/app/internal/app.go @@ -20,12 +20,9 @@ import ( "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/recover" "github.com/gofiber/fiber/v3/middleware/requestid" - "github.com/jackc/pgx/v5/pgxpool" ) -func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber.App, error) { - st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL) - +func New(cfg config.Config, st *store.Store, sessions *session.Store) (*fiber.App, error) { objects, err := objectstore.New(objectstore.Config{ Endpoint: cfg.S3Endpoint, Region: cfg.S3Region, @@ -74,6 +71,8 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber attachments.NewValidator(cfg.MaxImagePixels, cfg.ImageDecodeBytes), cfg.MaxUploadBytes, cfg.UploadDrainLimit, + cfg.MaxPendingJobs, + cfg.JobMaxAttempts, cfg.UploadTimeout, cfg.DownloadTimeout, cfg.UploadTempDir, diff --git a/app/internal/attachments/handler.go b/app/internal/attachments/handler.go index 6fb87491..0f8359bc 100644 --- a/app/internal/attachments/handler.go +++ b/app/internal/attachments/handler.go @@ -31,35 +31,31 @@ type Handler struct { validator *Validator maxUploadBytes int64 drainLimit int64 + maxPendingJobs int + jobMaxAttempts int uploadTimeout time.Duration downloadTimeout time.Duration tempDir string } -func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler { +func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, maxPendingJobs, jobMaxAttempts int, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler { return &Handler{ store: st, objects: client, validator: validator, maxUploadBytes: maxUploadBytes, drainLimit: drainLimit, + maxPendingJobs: maxPendingJobs, + jobMaxAttempts: jobMaxAttempts, uploadTimeout: uploadTimeout, downloadTimeout: downloadTimeout, tempDir: tempDir, } } -// Upload stores raw PNG/JPEG bytes. The order is deliberate: -// 1. authorize before reading the body; -// 2. stream the body to a bounded temporary file while hashing; -// 3. validate pixels and decode; -// 4. commit a pending metadata row; -// 5. PUT the object; -// 6. mark the row ready. -// -// A 201 is therefore returned only after both the object and its metadata are -// durable. Every failure between 4 and 6 has a compensation path, and the -// reconciler covers crashes that prevent compensation from running. +// Upload stages the original, then commits the attachment snapshot and its +// outbox job together. The 202 is therefore a promise about durable bytes plus +// durable intent, not a promise that resizing already happened. func (h *Handler) Upload(c fiber.Ctx) error { userID := common.MustUserID(c) @@ -88,6 +84,19 @@ func (h *Handler) Upload(c fiber.Ctx) error { return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg") } + // Backpressure: refuse new work before reading its bytes. The check is + // approximate across API replicas but bounds staging growth under overload. + if h.maxPendingJobs > 0 { + pending, countErr := h.store.CountPendingJobs(c.Context()) + if countErr != nil { + return countErr + } + if pending >= h.maxPendingJobs { + c.Set(fiber.HeaderRetryAfter, "5") + return common.Unavailable("processing queue is full") + } + } + // Reject an obviously oversized request before allocating a temp file or // decoding anything. Chunked requests are caught by the streaming counter // below. @@ -151,6 +160,7 @@ func (h *Handler) Upload(c fiber.Ctx) error { } attachmentID := uuid.New() + jobID := uuid.New() attachment := model.Attachment{ ID: attachmentID, ChatID: chatID, @@ -159,55 +169,51 @@ func (h *Handler) Upload(c fiber.Ctx) error { ContentType: info.ContentType, SizeBytes: written, SHA256: hex.EncodeToString(hasher.Sum(nil)), - Status: "pending", + Status: "queued", Variants: []model.Variant{}, ErrorCode: nil, CreatedAt: common.NowUTC(), UpdatedAt: common.NowUTC(), - // AGENT: the key is server-generated from UUIDs and never contains the - // client filename. That keeps traversal and collisions out of the - // object store even before application authorization is considered. - ObjectKey: objectKey(chatID, attachmentID), - } - - if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil { - return err + // AGENT: keys are server-generated from UUIDs, never the client + // filename. staging/ is the durable handoff prefix; the worker copies + // the same bytes to the final attachments/ key unchanged. + ObjectKey: objectKey(chatID, attachmentID), + StagingKey: stagingKey(chatID, attachmentID), + PipelineVersion: 1, } if _, err = file.Seek(0, 0); err != nil { - h.compensate(attachment.ID, attachment.ObjectKey) return common.BadRequest("cannot read upload body") } - if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil { - slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID) - h.compensate(attachment.ID, attachment.ObjectKey) + if err = h.objects.Put(ctx, attachment.StagingKey, file, written, attachment.ContentType); err != nil { + slog.Error("stage attachment object", "err", err, "attachment_id", attachment.ID) return common.Unavailable("object storage unavailable") } - updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID) - if err != nil { - slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID) - h.compensate(attachment.ID, attachment.ObjectKey) - return common.Unavailable("attachment metadata unavailable") + job := model.JobRecord{ + Job: model.Job{ + JobID: jobID, + AttachmentID: attachmentID, + PipelineVersion: attachment.PipelineVersion, + CreatedAt: attachment.CreatedAt, + }, + MaxAttempts: h.jobMaxAttempts, + } + if err = h.store.CreateQueuedAttachment(ctx, attachment, job); err != nil { + h.compensateStaging(attachment.StagingKey) + return err } - attachment.Status = "ready" - attachment.UpdatedAt = common.RFCFormTime(updatedAt) - return c.Status(fiber.StatusCreated).JSON(attachment) + return c.Status(fiber.StatusAccepted).JSON(attachment) } -// compensate is best effort by design. If either delete fails, the pending row -// or the object remains visible to the reconciler, which retries after the -// configured grace period. -func (h *Handler) compensate(attachmentID uuid.UUID, key string) { +// compensateStaging is best effort. If it fails, the reconciler removes an +// object with no metadata row after the grace period. +func (h *Handler) compensateStaging(key string) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) { - slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID) - } - if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil { - slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID) + slog.Error("compensate: delete staging object", "err", err) } } @@ -223,10 +229,9 @@ func (h *Handler) Get(c fiber.Ctx) error { if err != nil { return err } - if attachment.Status != "ready" { - return common.Conflict("attachment is not ready") - } + // queued/processing snapshots are part of lab5: the client polls this + // endpoint until status becomes ready or failed. return c.JSON(attachment) } @@ -242,7 +247,9 @@ func (h *Handler) Download(c fiber.Ctx) error { if variant == "" { variant = "original" } - if variant != "original" { + switch variant { + case "original", "thumbnail", "crop", "optimized": + default: return common.BadRequest("unsupported variant") } @@ -254,9 +261,28 @@ func (h *Handler) Download(c fiber.Ctx) error { return common.Conflict("attachment is not ready") } + key := attachment.ObjectKey + contentType := attachment.ContentType + sizeBytes := attachment.SizeBytes + if variant != "original" { + found := false + for _, candidate := range attachment.Variants { + if candidate.Name == variant { + key = store.VariantObjectKey(attachment.ObjectKey, candidate.Name) + contentType = candidate.ContentType + sizeBytes = candidate.SizeBytes + found = true + break + } + } + if !found { + return common.NotFound("attachment variant not found") + } + } + ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout) - body, size, err := h.objects.Get(ctx, attachment.ObjectKey) + body, size, err := h.objects.Get(ctx, key) if err != nil { cancel() if errors.Is(err, objectstore.ErrNotFound) { @@ -266,16 +292,16 @@ func (h *Handler) Download(c fiber.Ctx) error { slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID) return common.Unavailable("object storage unavailable") } - if size != attachment.SizeBytes { + if size != sizeBytes { _ = body.Close() cancel() slog.Error("attachment object size mismatch", - "attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size) + "attachment_id", attachment.ID, "metadata_size", sizeBytes, "object_size", size) return common.Unavailable("attachment content is inconsistent") } - c.Set(fiber.HeaderContentType, attachment.ContentType) - c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10)) + c.Set(fiber.HeaderContentType, contentType) + c.Set(fiber.HeaderContentLength, strconv.FormatInt(sizeBytes, 10)) c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename})) c.Set(fiber.HeaderCacheControl, "private, no-store") c.Set("X-Content-Type-Options", "nosniff") @@ -307,6 +333,10 @@ func objectKey(chatID, attachmentID uuid.UUID) string { return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID) } +func stagingKey(chatID, attachmentID uuid.UUID) string { + return fmt.Sprintf("staging/%s/%s", chatID, attachmentID) +} + // CleanupStaleUploads removes temp files abandoned by a previous crash. A // running request removes its own file in a defer; this only handles the // process-died-mid-upload case. diff --git a/app/internal/broker/broker.go b/app/internal/broker/broker.go new file mode 100644 index 00000000..e358c9b6 --- /dev/null +++ b/app/internal/broker/broker.go @@ -0,0 +1,253 @@ +// Package broker wraps RabbitMQ delivery and publisher confirms. PostgreSQL is +// still the source of truth: a message carries only a job id and a snapshot of +// the durable job intent. +package broker + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + amqp "github.com/rabbitmq/amqp091-go" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" +) + +// ErrRequeue asks the consumer to nack and requeue. The worker uses it only +// when its durable state could not be written (for example PostgreSQL is down). +var ErrRequeue = errors.New("requeue delivery") + +func dial(url string) (*amqp.Connection, error) { + return amqp.DialConfig(url, amqp.Config{ + Heartbeat: 10 * time.Second, + Locale: "en_US", + }) +} + +func declareQueue(ch *amqp.Channel, queue string) error { + // A quorum queue uses a durable replicated log on the broker. Delivery is + // still at least once; idempotency lives in the job effect key. + _, err := ch.QueueDeclare(queue, true, false, false, false, amqp.Table{ + "x-queue-type": "quorum", + }) + return err +} + +// Publisher is safe for concurrent use. It keeps one confirm-mode channel and +// reconnects lazily after a broker restart. +type Publisher struct { + url string + queue string + mu sync.Mutex + conn *amqp.Connection + ch *amqp.Channel +} + +func NewPublisher(url, queue string) *Publisher { + return &Publisher{url: url, queue: queue} +} + +func (p *Publisher) ensure() error { + if p.ch != nil && !p.ch.IsClosed() { + return nil + } + p.closeLocked() + conn, err := dial(p.url) + if err != nil { + return err + } + ch, err := conn.Channel() + if err != nil { + _ = conn.Close() + return err + } + if err = ch.Confirm(false); err != nil { + _ = ch.Close() + _ = conn.Close() + return err + } + if err = declareQueue(ch, p.queue); err != nil { + _ = ch.Close() + _ = conn.Close() + return err + } + p.conn, p.ch = conn, ch + return nil +} + +// Publish waits for the broker's publisher confirm. A nil return means the +// broker accepted the message, not that a worker executed it. +func (p *Publisher) Publish(ctx context.Context, job model.Job) error { + body, err := json.Marshal(job) + if err != nil { + return err + } + + p.mu.Lock() + defer p.mu.Unlock() + if err = p.ensure(); err != nil { + return err + } + + confirmation, err := p.ch.PublishWithDeferredConfirmWithContext(ctx, "", p.queue, true, false, amqp.Publishing{ + DeliveryMode: amqp.Persistent, + ContentType: "application/json", + MessageId: job.JobID.String(), + Timestamp: time.Now().UTC(), + Type: job.Type, + Body: body, + }) + if err != nil { + p.closeLocked() + return err + } + confirmed, err := confirmation.WaitContext(ctx) + if err != nil { + p.closeLocked() + return err + } + if !confirmed { + p.closeLocked() + return fmt.Errorf("publisher confirm was negative") + } + return nil +} + +func (p *Publisher) Close() { + p.mu.Lock() + defer p.mu.Unlock() + p.closeLocked() +} + +func (p *Publisher) closeLocked() { + if p.ch != nil { + _ = p.ch.Close() + } + if p.conn != nil { + _ = p.conn.Close() + } + p.ch, p.conn = nil, nil +} + +// Handler must return nil after the durable effect (and optional ack); the +// consumer acks then. ErrRequeue leaves the message for redelivery. +type Handler func(context.Context, model.Job) error + +type Consumer struct { + url string + queue string + logger *slog.Logger +} + +func NewConsumer(url, queue string, logger *slog.Logger) *Consumer { + if logger == nil { + logger = slog.Default() + } + return &Consumer{url: url, queue: queue, logger: logger} +} + +// Run reconnects until ctx is cancelled. concurrency bounds simultaneous +// handlers in this process; prefetch bounds unacked messages the broker may +// push. The two knobs are intentionally separate for the lab5 experiment. +func (c *Consumer) Run(ctx context.Context, concurrency, prefetch int, handler Handler) error { + for { + if ctx.Err() != nil { + return nil + } + err := c.consumeOnce(ctx, concurrency, prefetch, handler) + if ctx.Err() != nil { + return nil + } + c.logger.Error("rabbitmq consumer disconnected", "err", err) + select { + case <-ctx.Done(): + return nil + case <-time.After(2 * time.Second): + } + } +} + +func (c *Consumer) consumeOnce(ctx context.Context, concurrency, prefetch int, handler Handler) error { + conn, err := dial(c.url) + if err != nil { + return err + } + defer conn.Close() + + ch, err := conn.Channel() + if err != nil { + return err + } + defer ch.Close() + if err = declareQueue(ch, c.queue); err != nil { + return err + } + if err = ch.Qos(prefetch, 0, false); err != nil { + return err + } + deliveries, err := ch.Consume(c.queue, "", false, false, false, false, nil) + if err != nil { + return err + } + + sem := make(chan struct{}, concurrency) + var workers sync.WaitGroup + + stop := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + case <-stop: + } + _ = ch.Cancel("", false) + }() + + for { + select { + case <-ctx.Done(): + close(stop) + workers.Wait() + return nil + case delivery, ok := <-deliveries: + if !ok { + close(stop) + workers.Wait() + return errors.New("delivery channel closed") + } + var job model.Job + if err = json.Unmarshal(delivery.Body, &job); err != nil { + c.logger.Error("drop malformed job message", "err", err) + _ = delivery.Ack(false) + continue + } + + sem <- struct{}{} + workers.Add(1) + go func(delivery amqp.Delivery, job model.Job) { + defer workers.Done() + defer func() { <-sem }() + handlerErr := handler(ctx, job) + switch { + case handlerErr == nil: + if err := delivery.Ack(false); err != nil { + c.logger.Error("ack failed", "job_id", job.JobID, "err", err) + } + case errors.Is(handlerErr, ErrRequeue): + // A short sleep avoids a tight redelivery loop while a + // dependency is down. The lease/outbox still own the job. + time.Sleep(time.Second) + if err := delivery.Nack(false, true); err != nil { + c.logger.Error("nack failed", "job_id", job.JobID, "err", err) + } + default: + c.logger.Error("job handler failed", "job_id", job.JobID, "err", handlerErr) + _ = delivery.Ack(false) + } + }(delivery, job) + } + } +} diff --git a/app/internal/common/time.go b/app/internal/common/time.go index a6a4362a..6cab38f7 100644 --- a/app/internal/common/time.go +++ b/app/internal/common/time.go @@ -2,6 +2,7 @@ package common import ( "encoding/json" + "fmt" "time" ) @@ -22,6 +23,27 @@ func (t RFCFormTime) MarshalJSON() ([]byte, error) { return json.Marshal(utc.Format(layout)) } +// UnmarshalJSON accepts the same RFC 3339 strings MarshalJSON emits. The +// broker round-trips Job values through JSON, and a named time.Time type does +// not inherit encoding/json's time.Time special case on decode. +func (t *RFCFormTime) UnmarshalJSON(data []byte) error { + if string(data) == "null" { + return nil + } + + var raw string + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + parsed, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return fmt.Errorf("parse RFC form time: %w", err) + } + *t = RFCFormTime(parsed) + return nil +} + func NowUTC() RFCFormTime { return RFCFormTime(time.Now().UTC().Truncate(storageTimePrecision)) } diff --git a/app/internal/common/time_test.go b/app/internal/common/time_test.go index a177d48a..21fca560 100644 --- a/app/internal/common/time_test.go +++ b/app/internal/common/time_test.go @@ -43,6 +43,15 @@ func TestRFCFormTimeSerializesFractionalSeconds(t *testing.T) { if string(encoded) != tc.want { t.Fatalf("RFC form time = %s, want %s", encoded, tc.want) } + + var decoded RFCFormTime + if err = json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal RFC form time: %v", err) + } + want := tc.value.UTC().Truncate(storageTimePrecision) + if !time.Time(decoded).Equal(want) { + t.Fatalf("round trip = %v, want %v", time.Time(decoded), want) + } }) } } diff --git a/app/internal/config/config.go b/app/internal/config/config.go index 7b4aa123..3b71d110 100644 --- a/app/internal/config/config.go +++ b/app/internal/config/config.go @@ -9,6 +9,8 @@ import ( "strconv" "strings" "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" ) const ( @@ -60,6 +62,26 @@ type Config struct { ReconcileGrace time.Duration ReconcileInterval time.Duration + + // Lab5 asynchronous pipeline. BrokerURL is required by the API (outbox + // publisher) and the worker (consumer), but not by the reconciler. + BrokerURL string + WorkerOnly bool + JobQueue string + JobMaxAttempts int + JobTimeout time.Duration + JobLease time.Duration + RetryBase time.Duration + RetryMax time.Duration + WorkerConcurrency int + WorkerPrefetch int + MaxPendingJobs int + LeaseSweep time.Duration + StagingRetention time.Duration + WorkTempDir string + WorkerFaultMode string + CrashAfterEffect bool + CrashAfterClaim bool } func Load() (Config, error) { @@ -141,6 +163,75 @@ func Load() (Config, error) { } cfg.ReconcileInterval = time.Duration(reconcileInterval) * time.Second + if cfg.WorkerOnly, err = envBool("WORKER_ONLY", false); err != nil { + return Config{}, err + } + if cfg.CrashAfterEffect, err = envBool("WORKER_CRASH_AFTER_EFFECT", false); err != nil { + return Config{}, err + } + if cfg.CrashAfterClaim, err = envBool("WORKER_CRASH_AFTER_CLAIM", false); err != nil { + return Config{}, err + } + jobAttempts, err := envInt("JOB_MAX_ATTEMPTS", 5, 1, 100) + if err != nil { + return Config{}, err + } + jobTimeout, err := envInt("JOB_TIMEOUT_SECONDS", 60, 5, 3600) + if err != nil { + return Config{}, err + } + jobLease, err := envInt("JOB_LEASE_SECONDS", 90, 10, 7200) + if err != nil { + return Config{}, err + } + retryBase, err := envInt("JOB_RETRY_BASE_SECONDS", 2, 1, 3600) + if err != nil { + return Config{}, err + } + retryMax, err := envInt("JOB_RETRY_MAX_SECONDS", 30, 1, 7200) + if err != nil { + return Config{}, err + } + concurrency, err := envInt("WORKER_CONCURRENCY", 2, 1, 128) + if err != nil { + return Config{}, err + } + prefetch, err := envInt("WORKER_PREFETCH", 2, 1, 512) + if err != nil { + return Config{}, err + } + maxPending, err := envInt("MAX_PENDING_JOBS", 1000, 0, 0) + if err != nil { + return Config{}, err + } + leaseSweep, err := envInt("LEASE_SWEEP_SECONDS", 15, 1, 3600) + if err != nil { + return Config{}, err + } + stagingRetention, err := envInt("STAGING_RETENTION_SECONDS", 86400, 60, 0) + if err != nil { + return Config{}, err + } + + cfg.JobQueue = envOr("JOB_QUEUE", model.JobTypeAttachmentProcessV1) + cfg.JobMaxAttempts = jobAttempts + cfg.JobTimeout = time.Duration(jobTimeout) * time.Second + cfg.JobLease = time.Duration(jobLease) * time.Second + cfg.RetryBase = time.Duration(retryBase) * time.Second + cfg.RetryMax = time.Duration(retryMax) * time.Second + cfg.WorkerConcurrency = concurrency + cfg.WorkerPrefetch = prefetch + cfg.MaxPendingJobs = maxPending + cfg.LeaseSweep = time.Duration(leaseSweep) * time.Second + cfg.StagingRetention = time.Duration(stagingRetention) * time.Second + cfg.WorkTempDir = envOr("WORK_TEMP_DIR", filepath.Join(os.TempDir(), "messenger-work")) + cfg.WorkerFaultMode = envOr("WORKER_FAULT_MODE", "none") + switch cfg.WorkerFaultMode { + case "none", "transient-once", "permanent-once": + default: + return Config{}, fmt.Errorf("WORKER_FAULT_MODE must be none, transient-once or permanent-once") + } + if cfg.DatabaseURL, err = secret("DATABASE_URL"); err != nil { return Config{}, err } @@ -175,6 +266,17 @@ func Load() (Config, error) { return cfg, nil } + if cfg.BrokerURL, err = secret("BROKER_URL"); err != nil { + return Config{}, err + } + if cfg.BrokerURL == "" { + return Config{}, fmt.Errorf("BROKER_URL or BROKER_URL_FILE is required") + } + + if cfg.WorkerOnly { + return cfg, nil + } + if cfg.CacheURL, err = secret("CACHE_URL"); err != nil { return Config{}, err } diff --git a/app/internal/database/migrations/0004_async_jobs.down.sql b/app/internal/database/migrations/0004_async_jobs.down.sql new file mode 100644 index 00000000..8a635130 --- /dev/null +++ b/app/internal/database/migrations/0004_async_jobs.down.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS attachment_jobs; + +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_variants_array; +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_pipeline_version; +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state; +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status; + +-- Collapse asynchronous states back to lab4's single in-flight state. Ready +-- and failed rows keep their meaning; queued/processing work is abandoned. +UPDATE attachments +SET status = 'pending', error_code = NULL +WHERE status IN ('queued', 'processing'); + +ALTER TABLE attachments + DROP COLUMN IF EXISTS variants, + DROP COLUMN IF EXISTS pipeline_version, + DROP COLUMN IF EXISTS staging_key, + DROP COLUMN IF EXISTS ready_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status + CHECK (status IN ('pending', 'ready', 'failed')), + ADD CONSTRAINT attachments_error_state CHECK ( + (status = 'pending' AND error_code IS NULL) OR + (status = 'ready' AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) + ); diff --git a/app/internal/database/migrations/0004_async_jobs.up.sql b/app/internal/database/migrations/0004_async_jobs.up.sql new file mode 100644 index 00000000..36da6ff5 --- /dev/null +++ b/app/internal/database/migrations/0004_async_jobs.up.sql @@ -0,0 +1,76 @@ +-- Lab5 turns attachment processing into a durable asynchronous job. +-- The API stages bytes in S3, then in ONE transaction writes the attachment +-- snapshot and its outbox/job row. A separate publisher moves the intent to +-- RabbitMQ, so a broker outage cannot lose an accepted upload. + +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status; +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state; + +-- Lab4 "pending" meant "object PUT in progress". Nothing can finish it now, +-- because lab5 acceptors write "queued" plus a job. Preserve the row for the +-- operator instead of silently deleting it. +UPDATE attachments +SET status = 'failed', error_code = 'legacy_interrupted', updated_at = now() +WHERE status = 'pending'; + +ALTER TABLE attachments + ADD COLUMN IF NOT EXISTS staging_key text, + ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS variants jsonb NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS ready_at timestamptz; + +UPDATE attachments SET ready_at = updated_at WHERE status = 'ready' AND ready_at IS NULL; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD CONSTRAINT attachments_error_state CHECK ( + (status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) + ), + ADD CONSTRAINT attachments_variants_array + CHECK (jsonb_typeof(variants) = 'array'), + ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1); + +-- One job per logical attachment effect. published_at IS NULL is the outbox +-- marker: a committed intent that has not received a publisher confirm. +CREATE TABLE IF NOT EXISTS attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE, + type text NOT NULL, + pipeline_version integer NOT NULL, + attempt integer NOT NULL DEFAULT 0, + max_attempts integer NOT NULL, + status text NOT NULL, + error_code text, + available_at timestamptz NOT NULL, + published_at timestamptz, + lease_until timestamptz, + processing_started_at timestamptz, + finished_at timestamptz, + replay_count integer NOT NULL DEFAULT 0, + last_replay_at timestamptz, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachment_jobs_status + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_attempts CHECK (attempt >= 0 AND max_attempts >= 1), + CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_error_state CHECK ( + (status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) + ), + CONSTRAINT attachment_jobs_finished CHECK ( + (status IN ('queued', 'processing') AND finished_at IS NULL) OR + (status IN ('ready', 'failed') AND finished_at IS NOT NULL) + ) +); + +CREATE INDEX IF NOT EXISTS attachment_jobs_outbox_idx + ON attachment_jobs (available_at, created_at) + WHERE published_at IS NULL AND status = 'queued'; +CREATE INDEX IF NOT EXISTS attachment_jobs_lease_idx + ON attachment_jobs (lease_until) + WHERE status = 'processing'; +CREATE INDEX IF NOT EXISTS attachment_jobs_status_idx + ON attachment_jobs (status, created_at); diff --git a/app/internal/database/migrations/0005_retry_error_state.down.sql b/app/internal/database/migrations/0005_retry_error_state.down.sql new file mode 100644 index 00000000..bb36bf15 --- /dev/null +++ b/app/internal/database/migrations/0005_retry_error_state.down.sql @@ -0,0 +1,24 @@ +-- Returning to the stricter 0004 shape requires clearing retry reasons first. +UPDATE attachment_jobs +SET error_code = NULL +WHERE status IN ('queued', 'processing'); + +UPDATE attachments +SET error_code = NULL +WHERE status IN ('queued', 'processing'); + +ALTER TABLE attachments + DROP CONSTRAINT IF EXISTS attachments_error_state; +ALTER TABLE attachments + ADD CONSTRAINT attachments_error_state CHECK ( + (status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) + ); + +ALTER TABLE attachment_jobs + DROP CONSTRAINT IF EXISTS attachment_jobs_error_state; +ALTER TABLE attachment_jobs + ADD CONSTRAINT attachment_jobs_error_state CHECK ( + (status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) + ); diff --git a/app/internal/database/migrations/0005_retry_error_state.up.sql b/app/internal/database/migrations/0005_retry_error_state.up.sql new file mode 100644 index 00000000..2a1f35a8 --- /dev/null +++ b/app/internal/database/migrations/0005_retry_error_state.up.sql @@ -0,0 +1,23 @@ +-- ScheduleRetry records the transient reason on the job while it waits for the +-- next attempt. The original 0004 constraint only allowed error_code on failed +-- rows, so a jittered retry update was rejected and the broker redelivery was +-- then acknowledged as "busy", leaving the job processing until lease expiry. +-- Ready stays clean; failed stays explainable; queued/processing may carry the +-- most recent transient reason. +ALTER TABLE attachments + DROP CONSTRAINT IF EXISTS attachments_error_state; +ALTER TABLE attachments + ADD CONSTRAINT attachments_error_state CHECK ( + (status = 'ready' AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) OR + (status IN ('queued', 'processing')) + ); + +ALTER TABLE attachment_jobs + DROP CONSTRAINT IF EXISTS attachment_jobs_error_state; +ALTER TABLE attachment_jobs + ADD CONSTRAINT attachment_jobs_error_state CHECK ( + (status = 'ready' AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) OR + (status IN ('queued', 'processing')) + ); diff --git a/app/internal/database/migrations_test.go b/app/internal/database/migrations_test.go index bd1812b0..9380ad71 100644 --- a/app/internal/database/migrations_test.go +++ b/app/internal/database/migrations_test.go @@ -12,7 +12,7 @@ import ( // latestMigrationVersion is the only value to bump when a migration is added. // The rest of the test derives filenames and ordering from the directory. -const latestMigrationVersion = 3 +const latestMigrationVersion = 5 var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`) diff --git a/app/internal/model/job.go b/app/internal/model/job.go new file mode 100644 index 00000000..e46f619e --- /dev/null +++ b/app/internal/model/job.go @@ -0,0 +1,47 @@ +package model + +import ( + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "github.com/google/uuid" +) + +// Job is the broker payload described by contracts/jobs.schema.json. It never +// contains image bytes, bucket keys or credentials: the worker reloads trusted +// metadata from PostgreSQL by attachment_id. +type Job struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt common.RFCFormTime `json:"created_at"` +} + +const ( + JobTypeAttachmentProcessV1 = "attachment.process.v1" + + JobStatusQueued = "queued" + JobStatusProcessing = "processing" + JobStatusReady = "ready" + JobStatusFailed = "failed" +) + +// JobRecord is the durable outbox row plus the operator-visible job state. +type JobRecord struct { + Job + Status string + // AvailableAt is the earliest time the outbox may publish a retry. + AvailableAt time.Time + // PublishedAt nil means the outbox has not received a publisher confirm + // yet. This is the recovery marker for a crash between COMMIT and publish. + PublishedAt *time.Time + LeaseUntil *time.Time + ProcessingStartedAt *time.Time + FinishedAt *time.Time + MaxAttempts int + ErrorCode *string + ReplayCount int +} diff --git a/app/internal/model/model.go b/app/internal/model/model.go index b49abdcd..b4cae8a2 100644 --- a/app/internal/model/model.go +++ b/app/internal/model/model.go @@ -56,8 +56,10 @@ type Attachment struct { CreatedAt common.RFCFormTime `json:"created_at"` UpdatedAt common.RFCFormTime `json:"updated_at"` - // ObjectKey is internal and never serialized. - ObjectKey string `json:"-"` + // ObjectKey and StagingKey are internal and never serialized. + ObjectKey string `json:"-"` + StagingKey string `json:"-"` + PipelineVersion int `json:"-"` } func (a Attachment) GetID() uuid.UUID { return a.ID } diff --git a/app/internal/outbox/publisher.go b/app/internal/outbox/publisher.go new file mode 100644 index 00000000..f093356d --- /dev/null +++ b/app/internal/outbox/publisher.go @@ -0,0 +1,85 @@ +// Package outbox publishes committed job intents to RabbitMQ. The database row +// is the outbox: published_at IS NULL means "this intent still needs a broker +// confirm". A crash between COMMIT and publish is recovered by the next loop. +package outbox + +import ( + "context" + "fmt" + "log/slog" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" +) + +const batchSize = 20 + +type Publisher struct { + store *store.Store + broker *broker.Publisher + logger *slog.Logger +} + +func New(st *store.Store, b *broker.Publisher, logger *slog.Logger) *Publisher { + if logger == nil { + logger = slog.Default() + } + return &Publisher{store: st, broker: b, logger: logger} +} + +func (p *Publisher) Run(ctx context.Context) { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := p.PublishBatch(ctx); err != nil { + p.logger.Error("outbox publish pass failed", "err", err) + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + } + } +} + +func (p *Publisher) PublishBatch(ctx context.Context) error { + jobs, err := p.store.ListPublishableJobs(ctx, batchSize) + if err != nil { + return err + } + for _, record := range jobs { + if err = p.publish(ctx, record); err != nil { + return fmt.Errorf("publish job %s: %w", record.JobID, err) + } + } + return nil +} + +func (p *Publisher) publish(ctx context.Context, record model.JobRecord) error { + publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + message := model.Job{ + SchemaVersion: 1, + JobID: record.JobID, + Type: record.Type, + AttachmentID: record.AttachmentID, + PipelineVersion: record.PipelineVersion, + Attempt: record.Attempt + 1, + CreatedAt: record.CreatedAt, + } + if err := p.broker.Publish(publishCtx, message); err != nil { + return err + } + // Compare-and-set on attempt/status: a worker may have claimed it, and a + // retry may have been scheduled while the confirm was in flight. + return p.store.MarkJobPublished(publishCtx, record.JobID, record.Attempt, time.Now().UTC()) +} diff --git a/app/internal/pipeline/exif.go b/app/internal/pipeline/exif.go new file mode 100644 index 00000000..98182cd8 --- /dev/null +++ b/app/internal/pipeline/exif.go @@ -0,0 +1,162 @@ +package pipeline + +import ( + "encoding/binary" + "image" + "io" +) + +// jpegOrientation returns the EXIF orientation (1..8) of a JPEG, or 1 when the +// file has no usable EXIF block. Go's image/jpeg intentionally ignores EXIF, +// so the worker applies the transform after decoding. The original bytes are +// never rewritten. +func jpegOrientation(r io.ReadSeeker) int { + defer func() { _, _ = r.Seek(0, 0) }() + if _, err := r.Seek(0, 0); err != nil { + return 1 + } + + var soi [2]byte + if _, err := io.ReadFull(r, soi[:]); err != nil || soi[0] != 0xff || soi[1] != 0xd8 { + return 1 + } + + for { + var prefix [1]byte + if _, err := io.ReadFull(r, prefix[:]); err != nil || prefix[0] != 0xff { + return 1 + } + marker := byte(0xff) + for marker == 0xff { + if _, err := io.ReadFull(r, prefix[:]); err != nil { + return 1 + } + marker = prefix[0] + } + + // Markers without a length field, and start-of-scan where metadata + // after it cannot be an APP1 segment we care about. + if marker == 0xd8 || marker == 0xd9 || marker == 0xda || + marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7) { + return 1 + } + + var lengthBytes [2]byte + if _, err := io.ReadFull(r, lengthBytes[:]); err != nil { + return 1 + } + length := int(binary.BigEndian.Uint16(lengthBytes[:])) + if length < 2 { + return 1 + } + payloadLength := length - 2 + if marker != 0xe1 || payloadLength < 6 { + if _, err := r.Seek(int64(payloadLength), io.SeekCurrent); err != nil { + return 1 + } + continue + } + + payload := make([]byte, payloadLength) + if _, err := io.ReadFull(r, payload); err != nil { + return 1 + } + if string(payload[:6]) == "Exif\x00\x00" { + return parseTIFFOrientation(payload[6:]) + } + } +} + +func parseTIFFOrientation(data []byte) int { + if len(data) < 8 { + return 1 + } + var order binary.ByteOrder + switch string(data[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 1 + } + if order.Uint16(data[2:4]) != 42 { + return 1 + } + + ifdOffset := int(order.Uint32(data[4:8])) + if ifdOffset < 8 || ifdOffset+2 > len(data) { + return 1 + } + count := int(order.Uint16(data[ifdOffset : ifdOffset+2])) + position := ifdOffset + 2 + for index := 0; index < count; index++ { + if position+12 > len(data) { + return 1 + } + entry := data[position : position+12] + tag := order.Uint16(entry[0:2]) + fieldType := order.Uint16(entry[2:4]) + items := order.Uint32(entry[4:8]) + if tag == 0x0112 && fieldType == 3 && items >= 1 { + if items == 1 { + value := int(order.Uint16(entry[8:10])) + if value >= 1 && value <= 8 { + return value + } + return 1 + } + offset := int(order.Uint32(entry[8:12])) + if offset+2 > len(data) { + return 1 + } + value := int(order.Uint16(data[offset : offset+2])) + if value >= 1 && value <= 8 { + return value + } + return 1 + } + position += 12 + } + return 1 +} + +// orient applies the eight EXIF orientation transforms. Every branch returns a +// new image except orientation 1, so the caller can treat the result as +// read-only. +func orient(src image.Image, orientation int) image.Image { + if orientation <= 1 || orientation > 8 { + return src + } + bounds := src.Bounds() + width, height := bounds.Dx(), bounds.Dy() + swap := orientation >= 5 + outWidth, outHeight := width, height + if swap { + outWidth, outHeight = height, width + } + dst := image.NewRGBA(image.Rect(0, 0, outWidth, outHeight)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + var nx, ny int + switch orientation { + case 2: // mirror horizontal + nx, ny = width-1-x, y + case 3: // rotate 180 + nx, ny = width-1-x, height-1-y + case 4: // mirror vertical + nx, ny = x, height-1-y + case 5: // transpose + nx, ny = y, x + case 6: // rotate 90 clockwise + nx, ny = height-1-y, x + case 7: // transverse + nx, ny = height-1-y, width-1-x + case 8: // rotate 270 clockwise + nx, ny = y, width-1-x + } + dst.Set(nx, ny, src.At(bounds.Min.X+x, bounds.Min.Y+y)) + } + } + return dst +} diff --git a/app/internal/pipeline/image.go b/app/internal/pipeline/image.go new file mode 100644 index 00000000..1713a42c --- /dev/null +++ b/app/internal/pipeline/image.go @@ -0,0 +1,252 @@ +// Package pipeline turns one validated original image into the required lab5 +// variants. It is deterministic and side-effect free so a broker redelivery can +// simply overwrite the same keys with the same bytes. +package pipeline + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "image" + "image/color" + "image/jpeg" + _ "image/png" + "os" + + "github.com/gen2brain/webp" + "golang.org/x/image/draw" + "golang.org/x/sync/semaphore" +) + +const ( + ThumbnailMax = 256 + CropSize = 128 + OptimizedMax = 1280 + + // Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is + // copied byte-for-byte and keeps its metadata and checksum. + JPEGQuality = 85 + WebPQuality = 82 + WebPMethod = 4 +) + +type Processor struct { + maxPixels int64 + budgetBytes int64 + decodeBudget *semaphore.Weighted +} + +// Encoded is one variant ready to be uploaded. +type Encoded struct { + Name string + ContentType string + Bytes []byte + Width int + Height int + SHA256 string +} + +func New(maxPixels, decodeBudgetBytes int64) *Processor { + return &Processor{ + maxPixels: maxPixels, + budgetBytes: decodeBudgetBytes, + decodeBudget: semaphore.NewWeighted(decodeBudgetBytes), + } +} + +// Failure classifies pipeline errors for the retry policy. A corrupt image is +// permanent (retrying cannot help); I/O, lease expiry and dependency errors are +// transient. +type Failure struct { + Permanent bool + Code string + Err error +} + +func (f *Failure) Error() string { + if f.Err == nil { + return f.Code + } + return f.Code + ": " + f.Err.Error() +} + +func (f *Failure) Unwrap() error { return f.Err } + +func permanent(code string, err error) *Failure { + return &Failure{Permanent: true, Code: code, Err: err} +} + +func transient(code string, err error) *Failure { + return &Failure{Permanent: false, Code: code, Err: err} +} + +// Process decodes the file once and returns all variants in memory. Worker +// concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous +// decoded images, which is the memory-heavy part of the experiment. +func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) { + img, err := p.decode(ctx, file) + if err != nil { + return nil, err + } + + thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax)) + if err != nil { + return nil, permanent("processing_failed", err) + } + crop, err := renderJPEG("crop", centerSquare(img, CropSize)) + if err != nil { + return nil, permanent("processing_failed", err) + } + optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax)) + if err != nil { + return nil, permanent("processing_failed", err) + } + + return []Encoded{thumbnail, crop, optimized}, nil +} + +func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) { + if _, err := file.Seek(0, 0); err != nil { + return nil, transient("storage_read_failed", err) + } + cfg, format, err := image.DecodeConfig(file) + if err != nil { + return nil, permanent("invalid_image", err) + } + if format != "png" && format != "jpeg" { + return nil, permanent("invalid_image", fmt.Errorf("unsupported format %q", format)) + } + if cfg.Width <= 0 || cfg.Height <= 0 { + return nil, permanent("invalid_image", errors.New("empty image bounds")) + } + pixels := int64(cfg.Width) * int64(cfg.Height) + if pixels > p.maxPixels { + return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS")) + } + + // AGENT: the semaphore is acquired before Decode, not after. A decoder + // allocates roughly width*height*4 bytes; without this gate several 40 MP + // files can OOM the worker even with a small HTTP concurrency. + weight := pixels * 4 + if weight > p.budgetBytes { + return nil, permanent("invalid_image", errors.New("image exceeds decode memory budget")) + } + if err = p.decodeBudget.Acquire(ctx, weight); err != nil { + return nil, transient("processing_timeout", err) + } + defer p.decodeBudget.Release(weight) + + orientation := jpegOrientation(file) + if _, err = file.Seek(0, 0); err != nil { + return nil, transient("storage_read_failed", err) + } + img, decodeErr := safeDecode(file) + if decodeErr != nil { + return nil, permanent("invalid_image", decodeErr) + } + bounds := img.Bounds() + if bounds.Dx() != cfg.Width || bounds.Dy() != cfg.Height { + return nil, permanent("invalid_image", errors.New("decoded bounds differ from header")) + } + return orient(img, orientation), nil +} + +func safeDecode(file *os.File) (img image.Image, err error) { + defer func() { + if recovered := recover(); recovered != nil { + img = nil + err = fmt.Errorf("image decoder panic: %v", recovered) + } + }() + img, _, err = image.Decode(file) + return img, err +} + +// fitOnWhite scales down to fit maxWidth x maxHeight, never up. The source is +// composited over white, which is the contract for JPEG variants with alpha. +func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA { + bounds := src.Bounds() + width, height := fitSize(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight) + dst := image.NewRGBA(image.Rect(0, 0, width, height)) + draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src) + draw.CatmullRom.Scale(dst, dst.Bounds(), src, bounds, draw.Over, nil) + return dst +} + +// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the +// contract allows upscaling tiny sources for the crop variant. +func centerSquare(src image.Image, size int) *image.RGBA { + bounds := src.Bounds() + side := bounds.Dx() + if bounds.Dy() < side { + side = bounds.Dy() + } + x := bounds.Min.X + (bounds.Dx()-side)/2 + y := bounds.Min.Y + (bounds.Dy()-side)/2 + cropRect := image.Rect(x, y, x+side, y+side) + + dst := image.NewRGBA(image.Rect(0, 0, size, size)) + draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src) + draw.CatmullRom.Scale(dst, dst.Bounds(), src, cropRect, draw.Over, nil) + return dst +} + +// fitSize preserves aspect ratio, rounds down and clamps to at least 1 px. +// It deliberately never enlarges a small source. +func fitSize(width, height, maxWidth, maxHeight int) (int, int) { + if width <= maxWidth && height <= maxHeight { + return width, height + } + if width*maxHeight >= height*maxWidth { + newHeight := height * maxWidth / width + if newHeight < 1 { + newHeight = 1 + } + return maxWidth, newHeight + } + newWidth := width * maxHeight / height + if newWidth < 1 { + newWidth = 1 + } + return newWidth, maxHeight +} + +func renderJPEG(name string, img image.Image) (Encoded, error) { + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil { + return Encoded{}, err + } + bounds := img.Bounds() + return Encoded{ + Name: name, + ContentType: "image/jpeg", + Bytes: buf.Bytes(), + Width: bounds.Dx(), + Height: bounds.Dy(), + SHA256: digest(buf.Bytes()), + }, nil +} + +func renderWebP(img image.Image) (Encoded, error) { + var buf bytes.Buffer + if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil { + return Encoded{}, err + } + bounds := img.Bounds() + return Encoded{ + Name: "optimized", + ContentType: "image/webp", + Bytes: buf.Bytes(), + Width: bounds.Dx(), + Height: bounds.Dy(), + SHA256: digest(buf.Bytes()), + }, nil +} + +func digest(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/app/internal/pipeline/image_test.go b/app/internal/pipeline/image_test.go new file mode 100644 index 00000000..2a726ec7 --- /dev/null +++ b/app/internal/pipeline/image_test.go @@ -0,0 +1,177 @@ +package pipeline + +import ( + "bytes" + "context" + "encoding/binary" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "path/filepath" + "testing" +) + +func TestFitSizePreservesAspectAndNeverUpscales(t *testing.T) { + cases := []struct { + w, h, maxW, maxH int + wantW, wantH int + }{ + {320, 200, 256, 256, 256, 160}, + {200, 320, 256, 256, 160, 256}, + {100, 50, 256, 256, 100, 50}, + {4000, 1000, 1280, 1280, 1280, 320}, + {1000, 4000, 256, 256, 64, 256}, + {1, 4000, 256, 256, 1, 256}, + } + for _, tc := range cases { + w, h := fitSize(tc.w, tc.h, tc.maxW, tc.maxH) + if w != tc.wantW || h != tc.wantH { + t.Fatalf("fitSize(%d,%d,%d,%d) = %dx%d, want %dx%d", + tc.w, tc.h, tc.maxW, tc.maxH, w, h, tc.wantW, tc.wantH) + } + } +} + +func TestProcessProducesContractVariants(t *testing.T) { + file := pngFile(t, 320, 200, false) + variants, err := New(40_000_000, 256<<20).Process(context.Background(), file) + if err != nil { + t.Fatalf("Process: %v", err) + } + if len(variants) != 3 { + t.Fatalf("got %d variants, want 3", len(variants)) + } + + byName := map[string]Encoded{} + for _, variant := range variants { + byName[variant.Name] = variant + } + thumbnail := byName["thumbnail"] + if thumbnail.ContentType != "image/jpeg" || thumbnail.Width != 256 || thumbnail.Height != 160 { + t.Fatalf("thumbnail = %s %dx%d, want image/jpeg 256x160", + thumbnail.ContentType, thumbnail.Width, thumbnail.Height) + } + decoded, err := jpeg.Decode(bytes.NewReader(thumbnail.Bytes)) + if err != nil { + t.Fatalf("decode thumbnail: %v", err) + } + if decoded.Bounds().Dx() != 256 || decoded.Bounds().Dy() != 160 { + t.Fatalf("actual thumbnail bounds = %v", decoded.Bounds()) + } + + crop := byName["crop"] + if crop.ContentType != "image/jpeg" || crop.Width != 128 || crop.Height != 128 { + t.Fatalf("crop = %s %dx%d", crop.ContentType, crop.Width, crop.Height) + } + optimized := byName["optimized"] + if optimized.ContentType != "image/webp" || optimized.Width != 320 || optimized.Height != 200 { + t.Fatalf("optimized = %s %dx%d, want image/webp 320x200", optimized.ContentType, optimized.Width, optimized.Height) + } +} + +func TestThumbnailCompositesAlphaOnWhite(t *testing.T) { + file := pngFile(t, 64, 64, true) + variants, err := New(40_000_000, 256<<20).Process(context.Background(), file) + if err != nil { + t.Fatalf("Process: %v", err) + } + decoded, err := jpeg.Decode(bytes.NewReader(variants[0].Bytes)) + if err != nil { + t.Fatalf("decode thumbnail: %v", err) + } + r, g, b, _ := decoded.At(32, 32).RGBA() + if r < 0xf000 || g < 0xf000 || b < 0xf000 { + t.Fatalf("transparent pixel is not white: %04x %04x %04x", r, g, b) + } +} + +func TestEXIFOrientationRotatesBeforeResize(t *testing.T) { + file := jpegFileWithOrientation(t, 320, 200, 6) + if got := jpegOrientation(file); got != 6 { + t.Fatalf("jpegOrientation = %d, want 6", got) + } + variants, err := New(40_000_000, 256<<20).Process(context.Background(), file) + if err != nil { + t.Fatalf("Process: %v", err) + } + if variants[0].Width != 160 || variants[0].Height != 256 { + t.Fatalf("oriented thumbnail = %dx%d, want 160x256", variants[0].Width, variants[0].Height) + } +} + +func TestParseTIFFOrientationBigEndian(t *testing.T) { + tiff := []byte{ + 'M', 'M', 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x01, + 0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + if got := parseTIFFOrientation(tiff); got != 8 { + t.Fatalf("parseTIFFOrientation = %d, want 8", got) + } +} + +func pngFile(t *testing.T, width, height int, transparent bool) *os.File { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + alpha := uint8(0xff) + if transparent { + alpha = 0 + } + img.Set(x, y, color.RGBA{R: 10, G: 120, B: 200, A: alpha}) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode png: %v", err) + } + return tempFile(t, "image.png", buf.Bytes()) +} + +func jpegFileWithOrientation(t *testing.T, width, height int, orientation uint16) *os.File { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, width, height)) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, nil); err != nil { + t.Fatalf("encode jpeg: %v", err) + } + + payload := []byte{'E', 'x', 'i', 'f', 0, 0} + tiff := make([]byte, 26) + copy(tiff, []byte{'I', 'I'}) + binary.LittleEndian.PutUint16(tiff[2:4], 42) + binary.LittleEndian.PutUint32(tiff[4:8], 8) + binary.LittleEndian.PutUint16(tiff[8:10], 1) + binary.LittleEndian.PutUint16(tiff[10:12], 0x0112) + binary.LittleEndian.PutUint16(tiff[12:14], 3) + binary.LittleEndian.PutUint32(tiff[14:18], 1) + binary.LittleEndian.PutUint16(tiff[18:20], orientation) + payload = append(payload, tiff...) + + segment := append([]byte{0xff, 0xe1}, 0, 0) + binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2)) + segment = append(segment, payload...) + + data := append([]byte{}, buf.Bytes()[:2]...) + data = append(data, segment...) + data = append(data, buf.Bytes()[2:]...) + return tempFile(t, "oriented.jpg", data) +} + +func tempFile(t *testing.T, name string, data []byte) *os.File { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + t.Cleanup(func() { _ = file.Close() }) + return file +} diff --git a/app/internal/reconcile/reconcile.go b/app/internal/reconcile/reconcile.go index 6eabba08..ad568912 100644 --- a/app/internal/reconcile/reconcile.go +++ b/app/internal/reconcile/reconcile.go @@ -1,12 +1,11 @@ -// Package reconcile repairs the two failure windows between PostgreSQL -// metadata and S3 bytes: +// Package reconcile repairs the two storage/metadata gaps that a crash can +// leave behind: // -// - a pending metadata row whose upload never finished or whose 201 was -// never returned; -// - an S3 object whose metadata transaction never committed. +// - a staging object whose metadata row never committed; +// - a terminal attachment whose staging copy was not deleted yet. // -// Both repairs use a grace period so an upload that is still in flight is -// never touched. +// It never touches the final original or a variant while PostgreSQL still owns +// it. The grace period protects in-flight work. package reconcile import ( @@ -19,7 +18,7 @@ import ( "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" ) -const objectPrefix = "attachments/" +var prefixes = []string{"attachments/", "staging/"} type Reconciler struct { store *store.Store @@ -29,9 +28,9 @@ type Reconciler struct { } type Stats struct { - StalePending int `json:"stale_pending"` + StaleStaging int `json:"stale_staging"` OrphanObjects int `json:"orphan_objects"` - DeletedRows int `json:"deleted_rows"` + DeletedStaging int `json:"deleted_staging"` DeletedObjects int `json:"deleted_objects"` Skipped int `json:"skipped"` } @@ -47,68 +46,64 @@ func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) { var stats Stats cutoff := time.Now().UTC().Add(-r.grace) - // Phase 1: rows that were committed as pending but never reached ready. - stale, err := r.store.ListStalePendingAttachments(ctx, cutoff, 500) + // Phase 1: ready/failed rows eventually lose their staging copy. The + // worker usually deletes it immediately after ready commits; this covers a + // crash or a transient S3 error. + stale, err := r.store.ListStaleStagingAttachments(ctx, cutoff, 500) if err != nil { return stats, err } for _, attachment := range stale { - stats.StalePending++ + stats.StaleStaging++ if dryRun { - r.logger.Info("reconcile: would remove stale pending attachment", - "attachment_id", attachment.ID, "object_key", attachment.ObjectKey, - "created_at", attachment.CreatedAt) + r.logger.Info("reconcile: would remove stale staging", + "attachment_id", attachment.ID, "staging_key", attachment.StagingKey) continue } - - if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) { - // Keep the row for the next pass. Deleting it now would lose the - // only pointer to an object that might still exist. - r.logger.Error("reconcile: delete stale object", + if err = r.objects.Delete(ctx, attachment.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) { + r.logger.Error("reconcile: delete staging", "err", err, "attachment_id", attachment.ID) stats.Skipped++ continue } - if err = r.store.DeletePendingAttachment(ctx, attachment.ID); err != nil { - r.logger.Error("reconcile: delete stale row", "err", err, "attachment_id", attachment.ID) + if err = r.store.ClearStagingKey(ctx, attachment.ID); err != nil { + r.logger.Error("reconcile: clear staging key", "err", err, "attachment_id", attachment.ID) stats.Skipped++ continue } - stats.DeletedRows++ - stats.DeletedObjects++ + stats.DeletedStaging++ } - // Phase 2: objects with no metadata row at all. The pending row is written - // before the PUT in the normal path, so this only happens after a crash in - // a narrow window (or manual bucket tampering). - listed, err := r.objects.List(ctx, objectPrefix) + // Phase 2: anything under our prefixes with no database owner. The API + // writes staging before the metadata commit, so this window is real. + known, err := r.store.ListStorageKeys(ctx) if err != nil { return stats, err } - known, err := r.store.ListAttachmentObjectKeys(ctx) - if err != nil { - return stats, err - } - for _, object := range listed { - if _, ok := known[object.Key]; ok { - continue + for _, prefix := range prefixes { + listed, listErr := r.objects.List(ctx, prefix) + if listErr != nil { + return stats, listErr } - // A missing LastModified must not be treated as ancient: skipping is - // safer than deleting an object that may have just been created. - if object.LastModified.IsZero() || object.LastModified.After(cutoff) { - continue + for _, object := range listed { + if _, ok := known[object.Key]; ok { + continue + } + if object.LastModified.IsZero() || object.LastModified.After(cutoff) { + continue + } + stats.OrphanObjects++ + if dryRun { + r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key) + continue + } + if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) { + r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key) + stats.Skipped++ + continue + } + stats.DeletedObjects++ } - stats.OrphanObjects++ - if dryRun { - r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key) - continue - } - if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) { - r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key) - stats.Skipped++ - continue - } - stats.DeletedObjects++ } return stats, nil diff --git a/app/internal/store/attachments.go b/app/internal/store/attachments.go index cbb85d3c..22802999 100644 --- a/app/internal/store/attachments.go +++ b/app/internal/store/attachments.go @@ -3,6 +3,7 @@ package store import ( "context" "encoding/hex" + "encoding/json" "errors" "time" @@ -12,10 +13,11 @@ import ( "github.com/jackc/pgx/v5" ) -// CreatePendingAttachment records the intent to store an object before the S3 -// PUT happens. Committing this row first is what makes the reconciler able to -// distinguish "never uploaded" from "uploaded but process died". -func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachment) error { +// CreateQueuedAttachment is the durability boundary of the upload endpoint. +// The attachment snapshot and the outbox job commit together, so there is no +// observable state in which the API promised work but PostgreSQL lost the +// intent. Publication to RabbitMQ happens later and may be retried safely. +func (s *Store) CreateQueuedAttachment(ctx context.Context, att model.Attachment, job model.JobRecord) error { ctx, cancel := s.withTimeout(ctx) defer cancel() @@ -23,58 +25,74 @@ func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachmen if err != nil { return common.BadRequest("invalid sha256") } + if att.PipelineVersion < 1 { + att.PipelineVersion = 1 + } + variants, err := json.Marshal([]model.Variant{}) + if err != nil { + return common.Internal() + } - _, err = s.pool.Exec(ctx, ` + tx, err := s.pool.Begin(ctx) + if err != nil { + return dbError(err) + } + defer rollback(tx) + + _, err = tx.Exec(ctx, ` INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, status, error_code, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL, $9, $9)`, + object_key, staging_key, pipeline_version, variants, status, error_code, + created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued', NULL, $12, $12)`, att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.SizeBytes, - hash, att.ObjectKey, time.Time(att.CreatedAt)) - return dbError(err) -} - -// MarkAttachmentReady publishes metadata only after the object bytes are known -// to be in S3. A 201 is therefore a promise about both stores. -func (s *Store) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (time.Time, error) { - ctx, cancel := s.withTimeout(ctx) - defer cancel() - - updatedAt := time.Now().UTC() - tag, err := s.pool.Exec(ctx, ` - UPDATE attachments - SET status = 'ready', updated_at = $2 - WHERE id = $1 AND status = 'pending'`, id, updatedAt) + hash, att.ObjectKey, att.StagingKey, att.PipelineVersion, variants, time.Time(att.CreatedAt)) if err != nil { - return time.Time{}, dbError(err) + return dbError(err) } - if tag.RowsAffected() == 0 { - return time.Time{}, common.Conflict("attachment is not pending") + + _, err = tx.Exec(ctx, ` + INSERT INTO attachment_jobs + (job_id, attachment_id, type, pipeline_version, attempt, max_attempts, + status, error_code, available_at, published_at, lease_until, + processing_started_at, finished_at, replay_count, created_at, updated_at) + VALUES ($1, $2, $3, $4, 0, $5, 'queued', NULL, $6, NULL, NULL, + NULL, NULL, 0, $6, $6)`, + job.JobID, att.ID, model.JobTypeAttachmentProcessV1, att.PipelineVersion, + job.MaxAttempts, time.Time(att.CreatedAt)) + if err != nil { + return dbError(err) } - return updatedAt, nil + + if err = tx.Commit(ctx); err != nil { + return dbError(err) + } + return nil } -// DeletePendingAttachment removes a row only while it is still pending. The -// status guard is the safety property: if an upload completed concurrently, the -// compensation or reconciler pass must not delete a ready attachment. -func (s *Store) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error { +// CountPendingJobs is an admission control signal, not a queue size proof. +// Two API replicas may observe slightly stale counts, but the durable cap is +// still useful: it stops an overload from growing staging and PostgreSQL +// without limit. Rejections are 503 because the client may retry later. +func (s *Store) CountPendingJobs(ctx context.Context) (int, error) { ctx, cancel := s.withTimeout(ctx) defer cancel() - _, err := s.pool.Exec(ctx, - `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id) - return dbError(err) + var count int + err := s.pool.QueryRow(ctx, ` + SELECT count(*) FROM attachment_jobs + WHERE status IN ('queued', 'processing')`).Scan(&count) + if err != nil { + return 0, dbError(err) + } + return count, nil } func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID uuid.UUID) (model.Attachment, error) { ctx, cancel := s.withTimeout(ctx) defer cancel() - att, err := scanAttachment(s.pool.QueryRow(ctx, ` - SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, - a.size_bytes, a.sha256, a.status, a.error_code, - a.created_at, a.updated_at, a.object_key - FROM attachments a + att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+` JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2 WHERE a.id = $1`, attachmentID, userID)) if err != nil { @@ -83,20 +101,76 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID u return att, nil } -// ListStalePendingAttachments is the entry point for crash recovery. cutoff is -// an instant, not a duration: rows created before it are eligible. The caller -// computes it as now - grace so the boundary can be tuned and tested. -func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) { +// GetAttachment is the worker's trusted read. The worker never accepts bucket +// keys or filenames from the broker message. +func (s *Store) GetAttachment(ctx context.Context, attachmentID uuid.UUID) (model.Attachment, error) { ctx, cancel := s.withTimeout(ctx) defer cancel() - rows, err := s.pool.Query(ctx, ` - SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, - a.size_bytes, a.sha256, a.status, a.error_code, - a.created_at, a.updated_at, a.object_key - FROM attachments a - WHERE a.status = 'pending' AND a.created_at < $1 - ORDER BY a.created_at + att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1`, attachmentID)) + if err != nil { + return model.Attachment{}, err + } + return att, nil +} + +const attachmentSelect = ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, + a.created_at, a.updated_at, a.object_key, COALESCE(a.staging_key, ''), + a.pipeline_version, a.variants + FROM attachments a` + +func scanAttachment(row rowScanner) (model.Attachment, error) { + var ( + att model.Attachment + hash []byte + errCode *string + createdAt time.Time + updatedAt time.Time + variants []byte + ) + + if err := row.Scan( + &att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType, + &att.SizeBytes, &hash, &att.Status, &errCode, + &createdAt, &updatedAt, &att.ObjectKey, &att.StagingKey, + &att.PipelineVersion, &variants, + ); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, common.NotFound("attachment not found") + } + return model.Attachment{}, dbError(err) + } + + att.SHA256 = hex.EncodeToString(hash) + att.ErrorCode = errCode + att.CreatedAt = common.RFCFormTime(createdAt.UTC()) + att.UpdatedAt = common.RFCFormTime(updatedAt.UTC()) + att.Variants = []model.Variant{} + if len(variants) > 0 { + if err := json.Unmarshal(variants, &att.Variants); err != nil { + return model.Attachment{}, dbError(err) + } + } + if att.Variants == nil { + att.Variants = []model.Variant{} + } + return att, nil +} + +// ListStaleStagingAttachments returns terminal attachments whose staging copy +// may be removed. The grace period is applied to updated_at so a just-finished +// job is never cleaned while its commit is still settling. +func (s *Store) ListStaleStagingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + rows, err := s.pool.Query(ctx, attachmentSelect+` + WHERE a.status IN ('ready', 'failed') + AND a.updated_at < $1 + AND a.staging_key IS NOT NULL + ORDER BY a.updated_at LIMIT $2`, cutoff, limit) if err != nil { return nil, dbError(err) @@ -107,7 +181,7 @@ func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Tim for rows.Next() { att, scanErr := scanAttachment(rows) if scanErr != nil { - return nil, dbError(scanErr) + return nil, scanErr } result = append(result, att) } @@ -117,11 +191,24 @@ func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Tim return result, nil } -func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct{}, error) { +func (s *Store) ClearStagingKey(ctx context.Context, attachmentID uuid.UUID) error { ctx, cancel := s.withTimeout(ctx) defer cancel() - rows, err := s.pool.Query(ctx, `SELECT object_key FROM attachments`) + _, err := s.pool.Exec(ctx, + `UPDATE attachments SET staging_key = NULL WHERE id = $1`, attachmentID) + return dbError(err) +} + +// ListStorageKeys returns every object the database still owns. It includes +// variant keys derived from the original key; the reconciler must never delete +// a thumbnail that has no separate row. +func (s *Store) ListStorageKeys(ctx context.Context) (map[string]struct{}, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + rows, err := s.pool.Query(ctx, + `SELECT object_key, COALESCE(staging_key, ''), variants FROM attachments`) if err != nil { return nil, dbError(err) } @@ -129,11 +216,25 @@ func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct keys := make(map[string]struct{}) for rows.Next() { - var key string - if err = rows.Scan(&key); err != nil { + var ( + objectKey string + staging string + variants []byte + ) + if err = rows.Scan(&objectKey, &staging, &variants); err != nil { return nil, dbError(err) } - keys[key] = struct{}{} + keys[objectKey] = struct{}{} + if staging != "" { + keys[staging] = struct{}{} + } + var list []model.Variant + if err = json.Unmarshal(variants, &list); err != nil { + return nil, dbError(err) + } + for _, variant := range list { + keys[VariantObjectKey(objectKey, variant.Name)] = struct{}{} + } } if err = rows.Err(); err != nil { return nil, dbError(err) @@ -141,30 +242,9 @@ func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct return keys, nil } -func scanAttachment(row rowScanner) (model.Attachment, error) { - var ( - att model.Attachment - hash []byte - errCode *string - createdAt time.Time - updatedAt time.Time - ) - - if err := row.Scan( - &att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType, - &att.SizeBytes, &hash, &att.Status, &errCode, - &createdAt, &updatedAt, &att.ObjectKey, - ); err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return model.Attachment{}, common.NotFound("attachment not found") - } - return model.Attachment{}, dbError(err) - } - - att.SHA256 = hex.EncodeToString(hash) - att.ErrorCode = errCode - att.Variants = []model.Variant{} - att.CreatedAt = common.RFCFormTime(createdAt.UTC()) - att.UpdatedAt = common.RFCFormTime(updatedAt.UTC()) - return att, nil +// VariantObjectKey is the one place that names variant objects. Keeping it +// deterministic makes a retried job overwrite the same keys instead of +// creating a second variant set. +func VariantObjectKey(originalKey, variant string) string { + return originalKey + "/" + variant } diff --git a/app/internal/store/jobs.go b/app/internal/store/jobs.go new file mode 100644 index 00000000..f0224253 --- /dev/null +++ b/app/internal/store/jobs.go @@ -0,0 +1,433 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// ClaimResult tells the consumer whether this delivery owns the work. +type ClaimResult struct { + Job model.JobRecord + Attachment model.Attachment + // Terminal means the logical work is already ready/failed. Ack it. + Terminal bool + // Busy means another attempt owns an active lease, or a retry is not due. + // The publisher/lease sweeper will make the job visible again. + Busy bool +} + +// ClaimJob acquires or renews the logical job lease in one transaction. +// A broker redelivery of the same job_id therefore cannot become two workers +// writing results at the same time. +func (s *Store) ClaimJob(ctx context.Context, jobID uuid.UUID, lease time.Duration) (ClaimResult, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + tx, err := s.pool.Begin(ctx) + if err != nil { + return ClaimResult{}, dbError(err) + } + defer rollback(tx) + + job, err := scanJob(tx.QueryRow(ctx, ` + SELECT job_id, attachment_id, type, pipeline_version, attempt, max_attempts, + status, error_code, created_at, available_at, published_at, + lease_until, processing_started_at, finished_at, replay_count + FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID)) + if err != nil { + return ClaimResult{}, err + } + + att, err := scanAttachment(tx.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1 FOR UPDATE`, job.AttachmentID)) + if err != nil { + return ClaimResult{}, err + } + + now := time.Now().UTC() + if job.Status == model.JobStatusReady || job.Status == model.JobStatusFailed || att.Status == "ready" { + if err = tx.Commit(ctx); err != nil { + return ClaimResult{}, dbError(err) + } + return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil + } + if att.Status == "failed" { + if err = tx.Commit(ctx); err != nil { + return ClaimResult{}, dbError(err) + } + return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil + } + if job.Status == model.JobStatusProcessing && job.LeaseUntil != nil && job.LeaseUntil.After(now) { + if err = tx.Commit(ctx); err != nil { + return ClaimResult{}, dbError(err) + } + return ClaimResult{Job: job, Attachment: att, Busy: true}, nil + } + if job.Status == model.JobStatusQueued && job.AvailableAt.After(now) { + if err = tx.Commit(ctx); err != nil { + return ClaimResult{}, dbError(err) + } + return ClaimResult{Job: job, Attachment: att, Busy: true}, nil + } + + if job.Attempt >= job.MaxAttempts { + _, failErr := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'failed', error_code = 'retry_exhausted', + finished_at = $2, lease_until = NULL, updated_at = $2 + WHERE job_id = $1`, jobID, now) + if failErr != nil { + return ClaimResult{}, dbError(failErr) + } + _, failErr = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', updated_at = $2 + WHERE id = $1 AND status <> 'ready'`, job.AttachmentID, now) + if failErr != nil { + return ClaimResult{}, dbError(failErr) + } + if err = tx.Commit(ctx); err != nil { + return ClaimResult{}, dbError(err) + } + return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil + } + + attempt := job.Attempt + 1 + leaseUntil := now.Add(lease) + _, err = tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'processing', attempt = $2, error_code = NULL, + lease_until = $3, processing_started_at = COALESCE(processing_started_at, $4), + published_at = COALESCE(published_at, $4), updated_at = $4 + WHERE job_id = $1`, jobID, attempt, leaseUntil, now) + if err != nil { + return ClaimResult{}, dbError(err) + } + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = $2 + WHERE id = $1 AND status IN ('queued', 'processing')`, job.AttachmentID, now) + if err != nil { + return ClaimResult{}, dbError(err) + } + if err = tx.Commit(ctx); err != nil { + return ClaimResult{}, dbError(err) + } + + job.Attempt = attempt + job.Status = model.JobStatusProcessing + job.ErrorCode = nil + job.LeaseUntil = &leaseUntil + if job.ProcessingStartedAt == nil { + job.ProcessingStartedAt = &now + } + att.Status = "processing" + att.ErrorCode = nil + att.UpdatedAt = common.RFCFormTime(now) + return ClaimResult{Job: job, Attachment: att}, nil +} + +// CompleteJob commits the ready snapshot only for the attempt/version that +// still owns the lease. An older duplicate can therefore never roll a newer +// result back or overwrite it. +func (s *Store) CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, variants []model.Variant) (bool, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + if variants == nil { + variants = []model.Variant{} + } + encoded, err := json.Marshal(variants) + if err != nil { + return false, common.Internal() + } + + tx, err := s.pool.Begin(ctx) + if err != nil { + return false, dbError(err) + } + defer rollback(tx) + + var attachmentID uuid.UUID + var pipelineVersion int + var status string + err = tx.QueryRow(ctx, ` + SELECT attachment_id, pipeline_version, status + FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID). + Scan(&attachmentID, &pipelineVersion, &status) + if errors.Is(err, pgx.ErrNoRows) { + return false, common.NotFound("job not found") + } + if err != nil { + return false, dbError(err) + } + if status != model.JobStatusProcessing { + return false, nil + } + + att, err := scanAttachment(tx.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1 FOR UPDATE`, attachmentID)) + if err != nil { + return false, err + } + if att.PipelineVersion != pipelineVersion || att.Status == "ready" || att.Status == "failed" { + return false, nil + } + + now := time.Now().UTC() + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, finished_at = $3, + lease_until = NULL, updated_at = $3 + WHERE job_id = $1 AND attempt = $2 AND status = 'processing'`, + jobID, attempt, now) + if err != nil { + return false, dbError(err) + } + if tag.RowsAffected() == 0 { + return false, nil + } + + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'ready', variants = $2, error_code = NULL, + ready_at = $3, updated_at = $3 + WHERE id = $1 AND pipeline_version = $4 AND status IN ('queued', 'processing')`, + attachmentID, encoded, now, pipelineVersion) + if err != nil { + return false, dbError(err) + } + if err = tx.Commit(ctx); err != nil { + return false, dbError(err) + } + return true, nil +} + +// ScheduleRetry keeps the job in the durable outbox with a jittered delay. +// The caller acks the broker only after this commit, so the retry intent is +// never lost between "try later" and ack. +func (s *Store) ScheduleRetry(ctx context.Context, jobID uuid.UUID, attempt int, code string, delay time.Duration) (bool, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + tx, err := s.pool.Begin(ctx) + if err != nil { + return false, dbError(err) + } + defer rollback(tx) + + now := time.Now().UTC() + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'queued', error_code = $3, available_at = $4, + published_at = NULL, lease_until = NULL, updated_at = $5 + WHERE job_id = $1 AND status = 'processing' AND attempt = $2`, + jobID, attempt, code, now.Add(delay), now) + if err != nil { + return false, dbError(err) + } + if tag.RowsAffected() == 0 { + return false, nil + } + + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = $2 + WHERE id = (SELECT attachment_id FROM attachment_jobs WHERE job_id = $1) + AND status <> 'ready'`, jobID, now) + if err != nil { + return false, dbError(err) + } + return true, dbError(tx.Commit(ctx)) +} + +// FailJob records a terminal reason. It is idempotent: a later stale attempt +// that no longer matches the current attempt is ignored. +func (s *Store) FailJob(ctx context.Context, jobID uuid.UUID, attempt int, code string) error { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + tx, err := s.pool.Begin(ctx) + if err != nil { + return dbError(err) + } + defer rollback(tx) + + now := time.Now().UTC() + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'failed', error_code = $3, finished_at = $4, + lease_until = NULL, updated_at = $4 + WHERE job_id = $1 AND status = 'processing' AND attempt = $2`, + jobID, attempt, code, now) + if err != nil { + return dbError(err) + } + if tag.RowsAffected() == 0 { + return nil + } + + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'failed', error_code = $2, updated_at = $3 + WHERE id = (SELECT attachment_id FROM attachment_jobs WHERE job_id = $1) + AND status <> 'ready'`, jobID, code, now) + if err != nil { + return dbError(err) + } + return dbError(tx.Commit(ctx)) +} + +// RecoverExpiredLeases is the stuck-processing guard. A dead worker leaves a +// processing row whose lease expires; the sweeper requeues it or marks it +// retry_exhausted. It never touches ready work. +func (s *Store) RecoverExpiredLeases(ctx context.Context) (int, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + tag, err := s.pool.Exec(ctx, ` + WITH expired AS ( + UPDATE attachment_jobs j + SET status = CASE WHEN j.attempt >= j.max_attempts THEN 'failed' ELSE 'queued' END, + error_code = CASE WHEN j.attempt >= j.max_attempts THEN 'retry_exhausted' ELSE NULL END, + available_at = now(), + published_at = NULL, + lease_until = NULL, + finished_at = CASE WHEN j.attempt >= j.max_attempts THEN now() ELSE NULL END, + updated_at = now() + WHERE j.status = 'processing' + AND j.lease_until IS NOT NULL + AND j.lease_until < now() + RETURNING j.attachment_id, j.status, j.error_code + ) + UPDATE attachments a + SET status = CASE WHEN e.status = 'failed' THEN 'failed' ELSE 'queued' END, + error_code = CASE WHEN e.status = 'failed' THEN e.error_code ELSE NULL END, + updated_at = now() + FROM expired e + WHERE a.id = e.attachment_id AND a.status <> 'ready'`) + if err != nil { + return 0, dbError(err) + } + return int(tag.RowsAffected()), nil +} + +// ListPublishableJobs is the outbox read side. It intentionally does not lock +// rows: two API replicas may publish the same message, and the idempotent +// claim/effect makes that duplicate harmless. +func (s *Store) ListPublishableJobs(ctx context.Context, limit int) ([]model.JobRecord, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + rows, err := s.pool.Query(ctx, ` + SELECT job_id, attachment_id, type, pipeline_version, attempt, max_attempts, + status, error_code, created_at, available_at, published_at, + lease_until, processing_started_at, finished_at, replay_count + FROM attachment_jobs + WHERE published_at IS NULL AND status = 'queued' AND available_at <= now() + ORDER BY available_at, created_at + LIMIT $1`, limit) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + + var jobs []model.JobRecord + for rows.Next() { + job, scanErr := scanJob(rows) + if scanErr != nil { + return nil, scanErr + } + jobs = append(jobs, job) + } + if err = rows.Err(); err != nil { + return nil, dbError(err) + } + return jobs, nil +} + +// MarkJobPublished uses status+attempt as a compare-and-set. If a worker has +// already claimed the job, or a retry has superseded this publish, the update +// is a no-op and the outbox will publish the current intent later. +func (s *Store) MarkJobPublished(ctx context.Context, jobID uuid.UUID, attempt int, at time.Time) error { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + _, err := s.pool.Exec(ctx, ` + UPDATE attachment_jobs + SET published_at = $3, updated_at = $3 + WHERE job_id = $1 AND attempt = $2 AND status = 'queued' AND published_at IS NULL`, + jobID, attempt, at.UTC()) + return dbError(err) +} + +// ReplayFailed is the explicit operator action. The status guard means a +// second click, or a replay after success, cannot duplicate a ready effect. +func (s *Store) ReplayFailed(ctx context.Context, attachmentID uuid.UUID) (bool, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + tx, err := s.pool.Begin(ctx) + if err != nil { + return false, dbError(err) + } + defer rollback(tx) + + now := time.Now().UTC() + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'queued', attempt = 0, error_code = NULL, + available_at = $2, published_at = NULL, lease_until = NULL, + processing_started_at = NULL, finished_at = NULL, + replay_count = replay_count + 1, last_replay_at = $2, updated_at = $2 + WHERE attachment_id = $1 AND status = 'failed' + AND EXISTS (SELECT 1 FROM attachments + WHERE id = $1 AND status = 'failed')`, + attachmentID, now) + if err != nil { + return false, dbError(err) + } + if tag.RowsAffected() == 0 { + return false, nil + } + + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = $2 + WHERE id = $1 AND status = 'failed'`, attachmentID, now) + if err != nil { + return false, dbError(err) + } + if err = tx.Commit(ctx); err != nil { + return false, dbError(err) + } + return true, nil +} + +func scanJob(row rowScanner) (model.JobRecord, error) { + var ( + job model.JobRecord + created time.Time + errCode *string + ) + if err := row.Scan( + &job.JobID, &job.AttachmentID, &job.Type, &job.PipelineVersion, + &job.Attempt, &job.MaxAttempts, &job.Status, &errCode, &created, + &job.AvailableAt, &job.PublishedAt, &job.LeaseUntil, + &job.ProcessingStartedAt, &job.FinishedAt, &job.ReplayCount, + ); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.JobRecord{}, common.NotFound("job not found") + } + return model.JobRecord{}, dbError(err) + } + job.SchemaVersion = 1 + job.CreatedAt = common.RFCFormTime(created.UTC()) + job.ErrorCode = errCode + return job, nil +} diff --git a/app/internal/store/messages.go b/app/internal/store/messages.go index 0864d8b3..b0406743 100644 --- a/app/internal/store/messages.go +++ b/app/internal/store/messages.go @@ -78,9 +78,11 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t return model.Message{}, dbError(scanErr) } found++ - if status != "ready" { + // Lab5: a queued/processing attachment is already linked to its + // chat and may be sent. Only a terminal failed one is refused. + if status == "failed" { rows.Close() - return model.Message{}, common.Conflict("attachment is not ready") + return model.Message{}, common.Conflict("attachment failed processing") } } if err = rows.Err(); err != nil { diff --git a/app/internal/worker/worker.go b/app/internal/worker/worker.go new file mode 100644 index 00000000..f15dea05 --- /dev/null +++ b/app/internal/worker/worker.go @@ -0,0 +1,298 @@ +// Package worker runs the asynchronous attachment pipeline. It consumes one +// logical job, reads trusted metadata from PostgreSQL, loads the staged object, +// writes the original and variants, commits ready atomically, and only then +// lets the broker ack the message. +package worker + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "sync/atomic" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/pipeline" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" +) + +type Worker struct { + cfg config.Config + store *store.Store + objects *objectstore.Client + pipeline *pipeline.Processor + logger *slog.Logger + faultOnce atomic.Bool +} + +func New(cfg config.Config, st *store.Store, objects *objectstore.Client, logger *slog.Logger) *Worker { + if logger == nil { + logger = slog.Default() + } + w := &Worker{ + cfg: cfg, + store: st, + objects: objects, + pipeline: pipeline.New(cfg.MaxImagePixels, cfg.ImageDecodeBytes), + logger: logger, + } + w.faultOnce.Store(true) + return w +} + +// StartLeaseSweeper recovers a worker that died after claiming a job. Without +// this, the database would show "processing" forever and no message would be +// left to redeliver. +func (w *Worker) StartLeaseSweeper(ctx context.Context) { + ticker := time.NewTicker(w.cfg.LeaseSweep) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + count, err := w.store.RecoverExpiredLeases(ctx) + if err != nil { + w.logger.Error("lease sweep failed", "err", err) + } else if count > 0 { + w.logger.Warn("recovered expired job leases", "count", count) + } + } + } +} + +// Handle is the broker callback. Returning nil means "durable effect or +// deliberate no-op; ack". ErrRequeue means the database itself was unavailable. +func (w *Worker) Handle(ctx context.Context, message model.Job) error { + claim, err := w.store.ClaimJob(ctx, message.JobID, w.cfg.JobLease) + if err != nil { + if common.IsCode(err, common.ErrorCodeNotFound) { + w.logger.Warn("job no longer exists; acking", "job_id", message.JobID) + return nil + } + w.logger.Error("claim job failed", "job_id", message.JobID, "err", err) + return broker.ErrRequeue + } + if claim.Terminal || claim.Busy { + w.logger.Info("job is already finished or owned", + "job_id", message.JobID, "terminal", claim.Terminal, "busy", claim.Busy, + "attempt", claim.Job.Attempt) + return nil + } + if claim.Attachment.ID != message.AttachmentID { + w.logger.Error("message attachment mismatch; acking poison message", + "job_id", message.JobID, "message_attachment", message.AttachmentID, + "database_attachment", claim.Attachment.ID) + return nil + } + + log := w.logger.With( + "job_id", claim.Job.JobID, + "attachment_id", claim.Attachment.ID, + "attempt", claim.Job.Attempt, + "image_bytes", claim.Attachment.SizeBytes, + ) + if w.cfg.CrashAfterClaim { + // Deterministic fault injection for the "claimed but never finished" + // window. The lease sweeper requeues the durable job after it expires; + // it does not depend on the broker redelivery still being around. + w.logger.Error("fault injection: exiting after claim before processing", + "job_id", claim.Job.JobID, "lease_until", claim.Job.LeaseUntil) + os.Exit(10) + } + + log.Info("processing attachment") + + jobCtx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout) + defer cancel() + + if err = w.process(jobCtx, claim, log); err != nil { + if jobCtx.Err() != nil && errors.Is(jobCtx.Err(), context.DeadlineExceeded) { + err = &pipeline.Failure{Permanent: false, Code: "processing_timeout", Err: err} + } + return w.handleFailure(ctx, claim, err, log) + } + + log.Info("attachment ready") + if w.cfg.CrashAfterEffect { + // Deterministic fault injection for the "written but not acked" window. + // The unacked delivery is redelivered after the connection dies. + w.logger.Error("fault injection: exiting after durable effect before ack", + "job_id", claim.Job.JobID) + os.Exit(9) + } + return nil +} + +func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog.Logger) error { + if mode := w.cfg.WorkerFaultMode; mode != "none" && w.faultOnce.CompareAndSwap(true, false) { + if mode == "transient-once" { + return &pipeline.Failure{Permanent: false, Code: "processing_failed", + Err: errors.New("injected transient failure")} + } + return &pipeline.Failure{Permanent: true, Code: "invalid_image", + Err: errors.New("injected permanent failure")} + } + + att := claim.Attachment + if att.StagingKey == "" { + return &pipeline.Failure{Permanent: true, Code: "staging_missing", + Err: errors.New("attachment has no staging key")} + } + + reader, size, err := w.objects.Get(ctx, att.StagingKey) + if err != nil { + if errors.Is(err, objectstore.ErrNotFound) { + return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: err} + } + return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} + } + defer reader.Close() + if size != att.SizeBytes { + return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", + Err: fmt.Errorf("staged size %d != metadata %d", size, att.SizeBytes)} + } + + if err = os.MkdirAll(w.cfg.WorkTempDir, 0o700); err != nil { + return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} + } + file, err := os.CreateTemp(w.cfg.WorkTempDir, "job-*.part") + if err != nil { + return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} + } + path := file.Name() + defer func() { + _ = file.Close() + _ = os.Remove(path) + }() + + hasher := sha256.New() + // The same 10 MiB contract limit protects the worker from a corrupt row + // pointing at an unexpectedly large object. + written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, w.cfg.MaxUploadBytes+1), make([]byte, 32*1024)) + if err != nil { + return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} + } + if written > w.cfg.MaxUploadBytes { + return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("staged object too large")} + } + if written != att.SizeBytes || hex.EncodeToString(hasher.Sum(nil)) != att.SHA256 { + return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")} + } + + encoded, err := w.pipeline.Process(ctx, file) + if err != nil { + return err + } + + // Original bytes are copied unchanged from staging to the final key. + if _, err = file.Seek(0, 0); err != nil { + return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} + } + if err = w.objects.Put(ctx, att.ObjectKey, file, att.SizeBytes, att.ContentType); err != nil { + return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} + } + + variants := make([]model.Variant, 0, len(encoded)) + for _, item := range encoded { + key := store.VariantObjectKey(att.ObjectKey, item.Name) + if err = w.objects.Put(ctx, key, bytes.NewReader(item.Bytes), int64(len(item.Bytes)), item.ContentType); err != nil { + return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} + } + variants = append(variants, model.Variant{ + Name: item.Name, + ContentType: item.ContentType, + SizeBytes: int64(len(item.Bytes)), + Width: item.Width, + Height: item.Height, + SHA256: item.SHA256, + }) + } + + committed, err := w.store.CompleteJob(ctx, claim.Job.JobID, claim.Job.Attempt, variants) + if err != nil { + return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} + } + if !committed { + log.Warn("ready snapshot was superseded; leaving newer state alone") + return nil + } + + // Staging is removed only after ready commits. A crash before this delete + // leaves an object the reconciler will clean after retention. + if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) { + log.Warn("staging delete failed; reconciler will retry", "err", err) + return nil + } + if err = w.store.ClearStagingKey(ctx, att.ID); err != nil { + log.Warn("clear staging key failed; reconciler will retry", "err", err) + } + return nil +} + +func (w *Worker) handleFailure(ctx context.Context, claim store.ClaimResult, err error, log *slog.Logger) error { + var failure *pipeline.Failure + if !errors.As(err, &failure) { + failure = &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} + } + + if failure.Permanent { + log.Error("permanent job failure", "code", failure.Code, "err", failure.Err) + if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code); failErr != nil { + log.Error("record permanent failure failed; requeueing", "err", failErr) + return broker.ErrRequeue + } + return nil + } + + if claim.Job.Attempt >= claim.Job.MaxAttempts { + log.Error("retry budget exhausted", "code", failure.Code, "err", failure.Err) + if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, "retry_exhausted"); failErr != nil { + log.Error("record exhausted retry failed; requeueing", "err", failErr) + return broker.ErrRequeue + } + return nil + } + + delay := w.backoff(claim.Job.Attempt) + log.Warn("transient job failure; scheduling retry", + "code", failure.Code, "err", failure.Err, "retry_in", delay) + scheduled, retryErr := w.store.ScheduleRetry(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code, delay) + if retryErr != nil { + log.Error("schedule retry failed; requeueing", "err", retryErr) + return broker.ErrRequeue + } + if !scheduled { + // This attempt no longer owns the row (for example a lease sweep or a + // replay changed it). Requeue rather than ack a job whose future work + // was not durably recorded. + log.Warn("retry scheduling lost ownership; requeueing") + return broker.ErrRequeue + } + return nil +} + +// backoff is exponential with +/-50% jitter, capped by JOB_RETRY_MAX_SECONDS. +// Jitter prevents many retries from hitting a recovered dependency together. +func (w *Worker) backoff(attempt int) time.Duration { + delay := w.cfg.RetryBase + for i := 1; i < attempt && delay < w.cfg.RetryMax; i++ { + delay *= 2 + } + if delay > w.cfg.RetryMax { + delay = w.cfg.RetryMax + } + jitter := 0.5 + rand.Float64() // 0.5 .. 1.5 + return time.Duration(float64(delay) * jitter) +} diff --git a/compose.yaml b/compose.yaml index 8aa91626..76918681 100644 --- a/compose.yaml +++ b/compose.yaml @@ -24,6 +24,11 @@ x-api-environment: &api-environment MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} IMAGE_DECODE_MEMORY_MB: ${IMAGE_DECODE_MEMORY_MB:-256} UPLOAD_TEMP_DIR: /tmp/messenger-uploads + # Lab5 async pipeline. The API publishes outbox rows; the worker consumes. + BROKER_URL: "amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}@rabbitmq:5672/" + JOB_QUEUE: ${JOB_QUEUE:-attachment.process.v1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-1000} services: postgres: @@ -74,6 +79,29 @@ services: cpus: "${REDIS_CPU_LIMIT:-0.50}" memory: "${REDIS_MEM_LIMIT:-256M}" + rabbitmq: + image: rabbitmq:4.2-management-alpine + restart: unless-stopped + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env} + volumes: + - rabbitmqdata:/var/lib/rabbitmq + ports: + # Management UI is only for the local learning stand. + - "127.0.0.1:${RABBITMQ_MGMT_PORT:-15672}:15672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 3s + timeout: 5s + retries: 20 + start_period: 10s + deploy: + resources: + limits: + cpus: "${RABBITMQ_CPU_LIMIT:-1.0}" + memory: "${RABBITMQ_MEM_LIMIT:-512M}" + rustfs: image: rustfs/rustfs:1.0.4 restart: unless-stopped @@ -155,6 +183,8 @@ services: condition: service_healthy rustfs: condition: service_healthy + rabbitmq: + condition: service_healthy s3-init: condition: service_completed_successfully migrate: @@ -179,6 +209,53 @@ services: cpus: "${API_CPU_LIMIT:-0.50}" memory: "${API_MEM_LIMIT:-512M}" + worker: + <<: *api-build + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + rustfs: + condition: service_healthy + rabbitmq: + condition: service_healthy + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + environment: + <<: *api-environment + RUN_MIGRATIONS: "false" + WORKER_ONLY: "true" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-30} + LEASE_SWEEP_SECONDS: ${LEASE_SWEEP_SECONDS:-15} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + WORK_TEMP_DIR: /tmp/messenger-work + # Test hooks, off by default. They make the grade-4 failure windows + # deterministic; see evidence/ and REPORT.md. + WORKER_FAULT_MODE: ${WORKER_FAULT_MODE:-none} + WORKER_CRASH_AFTER_EFFECT: ${WORKER_CRASH_AFTER_EFFECT:-false} + WORKER_CRASH_AFTER_CLAIM: ${WORKER_CRASH_AFTER_CLAIM:-false} + entrypoint: ["/usr/local/bin/worker"] + healthcheck: + test: ["CMD-SHELL", "pgrep -f /usr/local/bin/worker >/dev/null"] + interval: 3s + timeout: 2s + retries: 10 + start_period: 5s + stop_grace_period: 20s + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: "${WORKER_CPU_LIMIT:-1.0}" + memory: "${WORKER_MEM_LIMIT:-1G}" + # AGENT: the reconciler uses the same scoped credential as the API. It never # touches ready attachments; it only removes stale pending rows/objects and # objects with no metadata row at all. @@ -256,3 +333,4 @@ volumes: pgdata: redisdata: rustfsdata: + rabbitmqdata: diff --git a/evidence/lab5-failover.txt b/evidence/lab5-failover.txt new file mode 100644 index 00000000..76821e22 --- /dev/null +++ b/evidence/lab5-failover.txt @@ -0,0 +1,11 @@ +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://172.18.0.1:8080 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://172.18.0.1:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 1, "first_recovery_seconds": 2.659} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +Operator scope evidence: +- before hook: docker compose ps showed two healthy api replicas. +- hook: scripts/stop-one.sh stopped mtusi2026lab-api-2 and left it stopped. +- during failover the load balancer served the surviving replica; the test observed 118 successful writes, 1 failed attempt and first recovery in 2.659 s. +- after the test: `docker start mtusi2026lab-api-2`; the replica returned to healthy, so the stand is restored to two API replicas. diff --git a/evidence/lab5-grade4.json b/evidence/lab5-grade4.json new file mode 100644 index 00000000..f17051cf --- /dev/null +++ b/evidence/lab5-grade4.json @@ -0,0 +1,208 @@ +{ + "generated_at": "2026-11-14T21:23:08.494056+00:00", + "fixture": { + "bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "width": 320, + "height": 200 + }, + "scenarios": { + "worker_stopped": { + "attachment_id": "3d1153db-d279-447c-8cda-e24fe2079119", + "queued_status": "queued", + "content_while_queued": 409, + "ready_after_worker_start": true, + "queue_seconds": 2.316, + "evidence": { + "original": { + "content_type": "image/png", + "size_bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "width": 320, + "height": 200 + }, + "thumbnail": { + "content_type": "image/jpeg", + "size_bytes": 5653, + "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", + "width": 256, + "height": 160 + }, + "crop": { + "content_type": "image/jpeg", + "size_bytes": 2441, + "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", + "width": 128, + "height": 128 + }, + "optimized": { + "content_type": "image/webp", + "size_bytes": 2576, + "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", + "width": 320, + "height": 200 + } + }, + "row": { + "attempt": 1, + "published": true, + "staging_cleared": true + } + }, + "broker_outage": { + "attachment_id": "bce30804-16d3-477c-a732-40bd336a0873", + "202_accepted_without_broker": true, + "unpublished_during_outage": true, + "api_ready_during_broker_outage": true, + "published_after_recovery": true, + "attempt": 1, + "queue_seconds": 9.005 + }, + "crash_after_effect": { + "attachment_id": "afc5a48d-7781-4627-a2b2-cf9be1d92ac5", + "worker_restart_count_before": 0, + "worker_restart_count_after": 1, + "ready_after_redelivery": true, + "variant_objects": 3, + "evidence": { + "original": { + "content_type": "image/png", + "size_bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "width": 320, + "height": 200 + }, + "thumbnail": { + "content_type": "image/jpeg", + "size_bytes": 5653, + "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", + "width": 256, + "height": 160 + }, + "crop": { + "content_type": "image/jpeg", + "size_bytes": 2441, + "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", + "width": 128, + "height": 128 + }, + "optimized": { + "content_type": "image/webp", + "size_bytes": 2576, + "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", + "width": 320, + "height": 200 + } + }, + "log_excerpt": [ + "worker-1 | time=2026-11-14T21:23:37.203Z level=INFO msg=\"processing attachment\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817", + "worker-1 | time=2026-11-14T21:23:37.392Z level=INFO msg=\"attachment ready\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817", + "worker-1 | time=2026-11-14T21:23:37.393Z level=ERROR msg=\"fault injection: exiting after durable effect before ack\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14", + "worker-1 | time=2026-11-14T21:23:37.932Z level=INFO msg=\"job is already finished or owned\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 terminal=true busy=false attempt=1" + ] + }, + "transient_retry": { + "attachment_id": "3b46dc74-f4cc-4c1b-89fc-f0e4023df603", + "attempts": 2, + "status": "ready", + "log_excerpt": [ + "worker-1 | time=2026-11-14T21:23:54.733Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817", + "worker-1 | time=2026-11-14T21:23:54.733Z level=WARN msg=\"transient job failure; scheduling retry\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817 code=processing_failed err=\"injected transient failure\" retry_in=2.25377422s", + "worker-1 | time=2026-11-14T21:23:57.029Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817", + "worker-1 | time=2026-11-14T21:23:57.219Z level=INFO msg=\"attachment ready\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817" + ] + }, + "permanent_failure_replay": { + "attachment_id": "a1417cf6-4d8b-48f7-9767-e5b876f0bdd1", + "failed_code": "invalid_image", + "failed_content_http": 409, + "failed_message_link_http": 409, + "replay_exit": 0, + "replay_output": [ + "2026/11/14 21:24:43 replay: attachment a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 requeued" + ], + "replay_count": 1, + "variant_objects_after_replay": 3, + "original_object_present": true, + "evidence": { + "original": { + "content_type": "image/png", + "size_bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "width": 320, + "height": 200 + }, + "thumbnail": { + "content_type": "image/jpeg", + "size_bytes": 5653, + "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", + "width": 256, + "height": 160 + }, + "crop": { + "content_type": "image/jpeg", + "size_bytes": 2441, + "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", + "width": 128, + "height": 128 + }, + "optimized": { + "content_type": "image/webp", + "size_bytes": 2576, + "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", + "width": 320, + "height": 200 + } + }, + "log_excerpt": [ + "worker-1 | time=2026-11-14T21:24:33.736Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817", + "worker-1 | time=2026-11-14T21:24:33.736Z level=ERROR msg=\"permanent job failure\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 code=invalid_image err=\"injected permanent failure\"", + "worker-1 | time=2026-11-14T21:24:43.247Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817", + "worker-1 | time=2026-11-14T21:24:43.469Z level=INFO msg=\"attachment ready\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817" + ] + }, + "lease_recovery": { + "attachment_id": "6d5091ae-474d-4e94-a4a9-f673aa16c9c1", + "one_shot_exit": 10, + "processing_before_expiry": true, + "attempts": 2, + "ready_after_sweep": true, + "evidence": { + "original": { + "content_type": "image/png", + "size_bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "width": 320, + "height": 200 + }, + "thumbnail": { + "content_type": "image/jpeg", + "size_bytes": 5653, + "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", + "width": 256, + "height": 160 + }, + "crop": { + "content_type": "image/jpeg", + "size_bytes": 2441, + "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", + "width": 128, + "height": 128 + }, + "optimized": { + "content_type": "image/webp", + "size_bytes": 2576, + "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", + "width": 320, + "height": 200 + } + }, + "log_excerpt": [ + "worker-1 | time=2026-11-14T21:24:51.033Z level=INFO msg=\"job is already finished or owned\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f terminal=false busy=true attempt=1", + "worker-1 | time=2026-11-14T21:24:59.019Z level=WARN msg=\"recovered expired job leases\" count=1", + "worker-1 | time=2026-11-14T21:24:59.229Z level=INFO msg=\"processing attachment\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817", + "worker-1 | time=2026-11-14T21:24:59.420Z level=INFO msg=\"attachment ready\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817" + ] + } + } +} diff --git a/evidence/lab5-grade4.md b/evidence/lab5-grade4.md new file mode 100644 index 00000000..6235aeff --- /dev/null +++ b/evidence/lab5-grade4.md @@ -0,0 +1,53 @@ +# Lab5 grade-4 runtime evidence + +Generated: `2026-11-14T21:23:08.494056+00:00` + +Synthetic fixture: 320x200 PNG, public contract fixture; no tokens or credentials are stored. + +| Scenario | Observation | +| --- | --- | +| Worker stopped during upload | 202 queued, content 409, message linked while queued, ready after worker start; queue 2.316 s, attempt 1 | +| Broker outage (COMMIT/publish gap) | 202 accepted with `published_at IS NULL`, API readiness stayed 200, job published and ready after RabbitMQ restart | +| Crash after effect, before ack | worker restart count 0 -> 1, 3 deterministic variant objects, ready | +| Transient error + jittered retry | failed once, attempt 2, ready | +| Permanent error + explicit replay | failed `invalid_image`, content/link 409, replay_count 1, ready with 3 variant objects | +| Crash after claim / expired lease | one-shot exit 10, processing, sweeper recovered, attempt 2, ready | + +## Sanitized log excerpts + +### crash_after_effect + +```text +worker-1 | time=2026-11-14T21:23:37.203Z level=INFO msg="processing attachment" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817 +worker-1 | time=2026-11-14T21:23:37.392Z level=INFO msg="attachment ready" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817 +worker-1 | time=2026-11-14T21:23:37.393Z level=ERROR msg="fault injection: exiting after durable effect before ack" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 +worker-1 | time=2026-11-14T21:23:37.932Z level=INFO msg="job is already finished or owned" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 terminal=true busy=false attempt=1 +``` + +### transient_retry + +```text +worker-1 | time=2026-11-14T21:23:54.733Z level=INFO msg="processing attachment" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817 +worker-1 | time=2026-11-14T21:23:54.733Z level=WARN msg="transient job failure; scheduling retry" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817 code=processing_failed err="injected transient failure" retry_in=2.25377422s +worker-1 | time=2026-11-14T21:23:57.029Z level=INFO msg="processing attachment" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817 +worker-1 | time=2026-11-14T21:23:57.219Z level=INFO msg="attachment ready" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817 +``` + +### permanent_failure_replay + +```text +worker-1 | time=2026-11-14T21:24:33.736Z level=INFO msg="processing attachment" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 +worker-1 | time=2026-11-14T21:24:33.736Z level=ERROR msg="permanent job failure" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 code=invalid_image err="injected permanent failure" +worker-1 | time=2026-11-14T21:24:43.247Z level=INFO msg="processing attachment" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 +worker-1 | time=2026-11-14T21:24:43.469Z level=INFO msg="attachment ready" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 +``` + +### lease_recovery + +```text +worker-1 | time=2026-11-14T21:24:51.033Z level=INFO msg="job is already finished or owned" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f terminal=false busy=true attempt=1 +worker-1 | time=2026-11-14T21:24:59.019Z level=WARN msg="recovered expired job leases" count=1 +worker-1 | time=2026-11-14T21:24:59.229Z level=INFO msg="processing attachment" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817 +worker-1 | time=2026-11-14T21:24:59.420Z level=INFO msg="attachment ready" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817 +``` + diff --git a/evidence/lab5-negative-cases.txt b/evidence/lab5-negative-cases.txt new file mode 100644 index 00000000..fd21ce70 --- /dev/null +++ b/evidence/lab5-negative-cases.txt @@ -0,0 +1 @@ +PASS: graded negative cases (413 byte/pixel, 415/404/400), no 5xx, cross-member download checksum matches diff --git a/evidence/lab5-persistence-containers.txt b/evidence/lab5-persistence-containers.txt new file mode 100644 index 00000000..caded5d6 --- /dev/null +++ b/evidence/lab5-persistence-containers.txt @@ -0,0 +1,38 @@ +# Persistence hook scope: container StartedAt before and after +$ docker compose ps --format "{{.Service}} {{.Status}}" +api Up 7 seconds (healthy) +api Up 7 seconds (healthy) +haproxy Up 7 seconds (healthy) +postgres Up 7 seconds (healthy) +rabbitmq Up 6 seconds (health: starting) +reconciler Up 5 minutes +redis Up 7 seconds (healthy) +rustfs Up 7 seconds (healthy) +worker Up 7 seconds (healthy) + +StartedAt before: +/mtusi2026lab-api-1 2026-11-14T21:25:41.77929768Z +/mtusi2026lab-api-2 2026-11-14T21:25:41.637807913Z +/mtusi2026lab-haproxy-1 2026-11-14T21:25:41.555990612Z +/mtusi2026lab-postgres-1 2026-11-14T21:25:41.820919775Z +/mtusi2026lab-rabbitmq-1 2026-11-14T21:25:42.573125104Z +/mtusi2026lab-reconciler-1 2026-11-14T21:19:51.25100733Z +/mtusi2026lab-redis-1 2026-11-14T21:25:41.663107884Z +/mtusi2026lab-rustfs-1 2026-11-14T21:25:41.292286108Z +/mtusi2026lab-worker-1 2026-11-14T21:25:41.248247598Z + +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://172.18.0.1:8080 +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://172.18.0.1:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +StartedAt after: +/mtusi2026lab-api-1 2026-11-14T21:25:53.837523775Z +/mtusi2026lab-api-2 2026-11-14T21:25:54.252693201Z +/mtusi2026lab-haproxy-1 2026-11-14T21:25:54.223324484Z +/mtusi2026lab-postgres-1 2026-11-14T21:25:54.026612072Z +/mtusi2026lab-rabbitmq-1 2026-11-14T21:25:55.048458301Z +/mtusi2026lab-reconciler-1 2026-11-14T21:19:51.25100733Z +/mtusi2026lab-redis-1 2026-11-14T21:25:54.361607562Z +/mtusi2026lab-rustfs-1 2026-11-14T21:25:53.897730396Z +/mtusi2026lab-worker-1 2026-11-14T21:25:53.965619121Z diff --git a/evidence/lab5-persistence.txt b/evidence/lab5-persistence.txt new file mode 100644 index 00000000..29d28d4c --- /dev/null +++ b/evidence/lab5-persistence.txt @@ -0,0 +1,4 @@ +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://172.18.0.1:8080 +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://172.18.0.1:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5-smoke.txt b/evidence/lab5-smoke.txt new file mode 100644 index 00000000..1fdb7772 --- /dev/null +++ b/evidence/lab5-smoke.txt @@ -0,0 +1,3 @@ +$ make test BASE_URL=http://172.18.0.1:8080 +python3 tests/smoke.py --base-url "http://172.18.0.1:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/rustfs/api-policy.json.template b/infra/rustfs/api-policy.json.template index 42c837ca..b95ad0b1 100644 --- a/infra/rustfs/api-policy.json.template +++ b/infra/rustfs/api-policy.json.template @@ -19,7 +19,8 @@ "s3:DeleteObject" ], "Resource": [ - "arn:aws:s3:::__BUCKET__/attachments/*" + "arn:aws:s3:::__BUCKET__/attachments/*", + "arn:aws:s3:::__BUCKET__/staging/*" ] } ] diff --git a/infra/rustfs/bootstrap.sh b/infra/rustfs/bootstrap.sh index 550bb419..5507a20a 100644 --- a/infra/rustfs/bootstrap.sh +++ b/infra/rustfs/bootstrap.sh @@ -13,6 +13,15 @@ set -eu : "${S3_APP_POLICY:?S3_APP_POLICY is required}" : "${S3_POLICY_TEMPLATE:?S3_POLICY_TEMPLATE is required}" +# AGENT: RustFS uses the user name as the access key, so the scoped identity +# the API signs with (S3_ACCESS_KEY_ID) must be the same string as S3_APP_USER. +# Divergence used to bootstrap successfully but every API readiness probe then +# saw 403 Forbidden, which is much harder to diagnose than a failed startup. +if [ "$S3_APP_USER" != "$S3_ACCESS_KEY_ID" ]; then + echo "S3_APP_USER ($S3_APP_USER) must equal S3_ACCESS_KEY_ID ($S3_ACCESS_KEY_ID): RustFS uses the user name as the access key" >&2 + exit 1 +fi + # AGENT: rc stores its alias file in $HOME. The image runs as an unprivileged # user, so point HOME at a writable temporary directory. export HOME="${HOME:-/tmp}" diff --git a/scripts/lab5_resilience.py b/scripts/lab5_resilience.py new file mode 100644 index 00000000..ef05577b --- /dev/null +++ b/scripts/lab5_resilience.py @@ -0,0 +1,761 @@ +#!/usr/bin/env python3 +"""Drive the lab5 grade-4 failure scenarios and write sanitized evidence. + +This is an operator script for the local Compose stand, not part of the public +black-box test suite. It deliberately restarts the worker, RabbitMQ and the API +replicas, so run it only against the disposable lab environment. + +Scenarios: + 1. worker stopped -> 202 queued -> message link -> worker start -> ready + 2. RabbitMQ down -> commit/outbox gap -> broker start -> ready + 3. crash after durable effect, before ack -> redelivery -> same result + 4. transient failure -> jittered retry -> ready on attempt 2 + 5. permanent failure -> failed registry -> explicit replay -> ready + 6. crash after claim -> lease expiry -> sweeper requeue -> ready +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env"] +WORKER = "mtusi2026lab-worker-1" + + +def run(cmd, *, env=None, timeout=180, check_code=True): + merged = os.environ.copy() + if env: + merged.update(env) + proc = subprocess.run( + cmd, + cwd=ROOT, + env=merged, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + if check_code and proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(shlex.quote(x) for x in cmd)}\n" + f"stdout:\n{proc.stdout[-4000:]}\nstderr:\n{proc.stderr[-4000:]}" + ) + return proc + + +def compose(*args, env=None, timeout=180, check_code=True): + return run(COMPOSE + list(args), env=env, timeout=timeout, check_code=check_code) + + +def compose_capture(*args, env=None, timeout=180): + return compose(*args, env=env, timeout=timeout).stdout + + +def psql(sql): + command = ( + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" -v ON_ERROR_STOP=1 -c ' + + shlex.quote(sql) + ) + out = compose_capture("exec", "-T", "postgres", "sh", "-c", command) + return [line for line in out.splitlines() if line.strip()] + + +def parse_ts(value): + value = value.strip() + if value.endswith("+00"): + value += ":00" + return datetime.fromisoformat(value.replace(" ", "T")) + + +def inspect(container_id): + try: + out = run(["docker", "inspect", container_id]).stdout + return json.loads(out)[0] + except (RuntimeError, json.JSONDecodeError): + return {} + + +def service_ids(service): + ids = compose_capture("ps", "-q", service).split() + return [item for item in ids if item] + + +def wait_healthy(service, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + ids = service_ids(service) + if ids: + states = [inspect(item) for item in ids] + if all( + state.get("State", {}).get("Health", {}).get("Status") == "healthy" + for state in states + ): + return ids + time.sleep(1) + raise Failure(f"{service} did not become healthy within {timeout}s") + + +def wait_stopped(service, timeout=30): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not service_ids(service): + return + time.sleep(0.5) + raise Failure(f"{service} did not stop within {timeout}s") + + +def recreate_worker(extra_env=None): + compose( + "up", + "-d", + "--no-build", + "--force-recreate", + "worker", + env=extra_env or {}, + timeout=240, + ) + wait_healthy("worker") + + +def recreate_api(extra_env=None): + compose( + "up", + "-d", + "--no-build", + "--force-recreate", + "api", + env=extra_env or {}, + timeout=240, + ) + wait_healthy("api") + # HAProxy resolves api through Docker DNS; give its health checks a moment + # to converge after both replica addresses change. + time.sleep(3) + + +def restore_worker(): + recreate_worker() + + +def restore_api(): + recreate_api() + + +def job_row(attachment_id): + rows = psql( + "SELECT j.job_id, j.status, j.attempt, j.error_code, j.created_at, " + "j.processing_started_at, j.finished_at, j.published_at, j.replay_count, " + "a.staging_key, a.object_key " + "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id " + f"WHERE j.attachment_id = '{attachment_id}'" + ) + if len(rows) != 1: + raise Failure(f"expected one job row for {attachment_id}, got {len(rows)}") + parts = rows[0].split("|") + return { + "job_id": parts[0], + "status": parts[1], + "attempt": int(parts[2]), + "error_code": None if parts[3] == "" else parts[3], + "created_at": parse_ts(parts[4]), + "processing_started_at": None if parts[5] == "" else parse_ts(parts[5]), + "finished_at": None if parts[6] == "" else parse_ts(parts[6]), + "published_at": None if parts[7] == "" else parse_ts(parts[7]), + "replay_count": int(parts[8]), + "staging_key": parts[9], + "object_key": parts[10], + } + + +def wait_job_status(attachment_id, wanted, timeout=90): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = job_row(attachment_id) + if last["status"] == wanted: + return last + if wanted != "failed" and last["status"] == "failed": + raise Failure(f"job {attachment_id} failed unexpectedly: {last['error_code']}") + time.sleep(0.4) + raise Failure(f"job {attachment_id} did not reach {wanted}; last={last}") + + +def s3_variant_keys(prefix): + script = ( + 'export HOME=/tmp; ' + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; ' + 'rc ls -r --json --no-progress "app/$S3_BUCKET/$PREFIX"' + ) + proc = compose( + "run", + "--rm", + "--no-deps", + "-e", + f"PREFIX={prefix}", + "--entrypoint", + "sh", + "s3-init", + "-c", + script, + check_code=False, + timeout=120, + ) + if proc.returncode != 0: + raise Failure(f"rc ls failed: {proc.stderr[-1000:]}") + payload = json.loads(proc.stdout) + return sorted(item["key"] for item in payload.get("items", [])) + + +def s3_object_exists(key): + script = ( + 'export HOME=/tmp; ' + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; ' + 'rc head --quiet "app/$S3_BUCKET/$KEY" >/dev/null' + ) + proc = compose( + "run", + "--rm", + "--no-deps", + "-e", + f"KEY={key}", + "--entrypoint", + "sh", + "s3-init", + "-c", + script, + check_code=False, + timeout=120, + ) + return proc.returncode == 0 + + +def jpeg_size(data): + if not data.startswith(b"\xff\xd8"): + raise Failure("variant is not a JPEG") + index = 2 + while index + 9 < len(data): + if data[index] != 0xFF: + index += 1 + continue + marker = data[index + 1] + index += 2 + if marker in (0xD8, 0xD9): + continue + length = int.from_bytes(data[index : index + 2], "big") + if marker in (0xC0, 0xC1, 0xC2): + height = int.from_bytes(data[index + 3 : index + 5], "big") + width = int.from_bytes(data[index + 5 : index + 7], "big") + return width, height + index += length + raise Failure("JPEG dimensions not found") + + +def png_size(data): + if not data.startswith(b"\x89PNG\r\n\x1a\n"): + raise Failure("variant is not a PNG") + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + return width, height + + +def verify_ready_attachment(client, actor, attachment, original): + check(attachment["status"] == "ready", "attachment is not ready") + check(attachment["error_code"] is None, "ready attachment has an error_code") + variants = {item["name"]: item for item in attachment["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "wrong variant set") + check( + (variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160), + "thumbnail is not 256x160 for the 320x200 fixture", + ) + check( + (variants["crop"]["width"], variants["crop"]["height"]) == (128, 128), + "crop is not 128x128", + ) + check( + (variants["optimized"]["width"], variants["optimized"]["height"]) == (320, 200), + "optimized variant upscaled the small fixture", + ) + + downloaded = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + actor=actor, + )["bytes"] + check(downloaded == original, "original download differs from the upload") + check( + hashlib.sha256(downloaded).hexdigest() == attachment["sha256"], + "original download checksum differs", + ) + + checks = { + "original": (downloaded, "image/png", png_size), + "thumbnail": ( + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content?variant=thumbnail", + actor=actor, + )["bytes"], + "image/jpeg", + jpeg_size, + ), + "crop": ( + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content?variant=crop", + actor=actor, + )["bytes"], + "image/jpeg", + jpeg_size, + ), + "optimized": ( + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content?variant=optimized", + actor=actor, + )["bytes"], + "image/webp", + None, + ), + } + observed = {} + for name, (data, media_type, size_parser) in checks.items(): + if media_type == "image/jpeg": + check(size_parser(data) == ( + variants[name]["width"], + variants[name]["height"], + ), f"{name} bytes do not match metadata dimensions") + if media_type == "image/webp": + check(data.startswith(b"RIFF") and data[8:12] == b"WEBP", "optimized is not WebP") + if name != "original": + check( + hashlib.sha256(data).hexdigest() == variants[name]["sha256"], + f"{name} checksum differs", + ) + check(len(data) == variants[name]["size_bytes"], f"{name} size differs") + observed[name] = { + "content_type": media_type, + "size_bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "width": variants[name]["width"] if name != "original" else 320, + "height": variants[name]["height"] if name != "original" else 200, + } + return observed + + +def upload(client, actor, chat, content, name): + return client.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename={name}", + 202, + actor=actor, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + + +def attach_message(client, actor, chat, attachment_id, expected=201): + return client.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + expected, + actor=actor, + body={"text": "", "attachment_ids": [attachment_id]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + +def worker_logs(): + return compose_capture("logs", "--no-color", "--tail=160", "worker") + + +def relevant_logs(text, tokens): + lines = [] + for line in text.splitlines(): + if any(token and token in line for token in tokens): + lines.append(line) + return lines + + +def scenario_worker_stopped(client, actor, chat, content, evidence): + compose("stop", "worker") + wait_stopped("worker") + attachment = upload(client, actor, chat, content, "grade4-worker-stopped.png") + check(attachment["status"] == "queued", "upload did not return queued") + attach_message(client, actor, chat, attachment["id"]) + content_status = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + 409, + actor=actor, + )["json"] + check(content_status["error"]["code"] == "conflict", "queued content is not 409") + compose("start", "worker") + wait_healthy("worker") + ready = client.wait_ready(actor, attachment, wait_seconds=60) + observed = verify_ready_attachment(client, actor, ready, content) + row = job_row(attachment["id"]) + check(row["status"] == "ready" and row["staging_key"] == "", "ready row/staging cleanup mismatch") + evidence["worker_stopped"] = { + "attachment_id": attachment["id"], + "queued_status": attachment["status"], + "content_while_queued": 409, + "ready_after_worker_start": True, + "queue_seconds": round( + (row["processing_started_at"] - row["created_at"]).total_seconds(), 3 + ), + "evidence": observed, + "row": { + "attempt": row["attempt"], + "published": row["published_at"] is not None, + "staging_cleared": row["staging_key"] == "", + }, + } + return attachment["id"] + + +def scenario_broker_outage(client, actor, chat, content, evidence): + compose("stop", "rabbitmq") + wait_stopped("rabbitmq") + attachment = upload(client, actor, chat, content, "grade4-broker-down.png") + row = job_row(attachment["id"]) + check(row["status"] == "queued", "job is not queued during broker outage") + check(row["published_at"] is None, "job unexpectedly published without broker") + time.sleep(3) + row_again = job_row(attachment["id"]) + check(row_again["published_at"] is None, "outbox published while broker was down") + healthy = client.call("GET", "/health/ready", schema="Health") + check(healthy["json"]["status"] == "ok", "API readiness depends on RabbitMQ") + compose("start", "rabbitmq") + wait_healthy("rabbitmq") + ready = client.wait_ready(actor, attachment, wait_seconds=90) + verify_ready_attachment(client, actor, ready, content) + row_ready = job_row(attachment["id"]) + check(row_ready["published_at"] is not None, "job was not published after broker recovery") + evidence["broker_outage"] = { + "attachment_id": attachment["id"], + "202_accepted_without_broker": attachment["status"] == "queued", + "unpublished_during_outage": True, + "api_ready_during_broker_outage": healthy["json"]["status"] == "ok", + "published_after_recovery": True, + "attempt": row_ready["attempt"], + "queue_seconds": round( + (row_ready["processing_started_at"] - row_ready["created_at"]).total_seconds(), 3 + ), + } + return attachment["id"] + + +def scenario_crash_after_effect(client, actor, chat, content, evidence): + recreate_worker({"WORKER_CRASH_AFTER_EFFECT": "true"}) + worker_id = service_ids("worker")[0] + restarts_before = inspect(worker_id)["RestartCount"] + attachment = upload(client, actor, chat, content, "grade4-crash-after-effect.png") + ready = client.wait_ready(actor, attachment, wait_seconds=90) + observed = verify_ready_attachment(client, actor, ready, content) + deadline = time.monotonic() + 30 + restarts_after = restarts_before + while time.monotonic() < deadline and restarts_after <= restarts_before: + time.sleep(0.5) + ids = service_ids("worker") + if not ids: + continue + restarts_after = inspect(ids[0]).get("RestartCount", restarts_after) + row = job_row(attachment["id"]) + keys = s3_variant_keys(row["object_key"]) + check(len(keys) == 3, f"expected exactly three variant objects, got {len(keys)}") + logs = relevant_logs( + worker_logs(), + [attachment["id"], row["job_id"], "fault injection", "attachment ready"], + ) + check(any("after durable effect before ack" in line for line in logs), "crash hook did not log") + evidence["crash_after_effect"] = { + "attachment_id": attachment["id"], + "worker_restart_count_before": restarts_before, + "worker_restart_count_after": restarts_after, + "ready_after_redelivery": True, + "variant_objects": len(keys), + "evidence": observed, + "log_excerpt": logs, + } + restore_worker() + return attachment["id"] + + +def scenario_transient_retry(client, actor, chat, content, evidence): + recreate_worker( + { + "WORKER_FAULT_MODE": "transient-once", + "JOB_RETRY_BASE_SECONDS": "2", + "JOB_RETRY_MAX_SECONDS": "8", + } + ) + attachment = upload(client, actor, chat, content, "grade4-transient-retry.png") + ready = client.wait_ready(actor, attachment, wait_seconds=90) + verify_ready_attachment(client, actor, ready, content) + row = job_row(attachment["id"]) + check(row["status"] == "ready", "transient retry did not finish ready") + check(row["attempt"] >= 2, "transient retry did not use a second attempt") + logs = relevant_logs( + worker_logs(), + [attachment["id"], row["job_id"], "transient job failure", "processing attachment"], + ) + check(any("transient job failure" in line for line in logs), "retry log missing") + evidence["transient_retry"] = { + "attachment_id": attachment["id"], + "attempts": row["attempt"], + "status": row["status"], + "log_excerpt": logs, + } + restore_worker() + return attachment["id"] + + +def scenario_permanent_failure_and_replay(client, actor, chat, content, evidence): + recreate_worker({"WORKER_FAULT_MODE": "permanent-once"}) + attachment = upload(client, actor, chat, content, "grade4-permanent-failure.png") + failed = wait_job_status(attachment["id"], "failed", timeout=45) + check(failed["error_code"] == "invalid_image", "unexpected permanent error code") + snapshot = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}", + actor=actor, + schema="Attachment", + )["json"] + check(snapshot["status"] == "failed", "attachment did not expose failed") + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + 409, + actor=actor, + ) + conflict = attach_message(client, actor, chat, attachment["id"], expected=409)["json"] + check(conflict["error"]["code"] == "conflict", "failed attachment was attachable") + failure_log = relevant_logs( + worker_logs(), + [attachment["id"], failed["job_id"], "permanent job failure", "invalid_image"], + ) + + restore_worker() + replay = compose( + "exec", + "-T", + "worker", + "/usr/local/bin/worker", + "-replay", + attachment["id"], + check_code=False, + ) + check(replay.returncode == 0, f"replay command failed: {replay.stderr[-1000:]}") + ready = client.wait_ready(actor, attachment, wait_seconds=90) + observed = verify_ready_attachment(client, actor, ready, content) + row = job_row(attachment["id"]) + check(row["status"] == "ready" and row["replay_count"] >= 1, "replay did not finish ready") + keys = s3_variant_keys(row["object_key"]) + check(len(keys) == 3, f"replay created a second variant set: {keys}") + check(s3_object_exists(row["object_key"]), "original object is missing after replay") + after_replay_log = relevant_logs( + worker_logs(), + [attachment["id"], row["job_id"], "replay", "attachment ready"], + ) + logs = failure_log + after_replay_log + evidence["permanent_failure_replay"] = { + "attachment_id": attachment["id"], + "failed_code": failed["error_code"], + "failed_content_http": 409, + "failed_message_link_http": 409, + "replay_exit": replay.returncode, + "replay_output": (replay.stdout + replay.stderr).strip().splitlines(), + "replay_count": row["replay_count"], + "variant_objects_after_replay": len(keys), + "original_object_present": True, + "evidence": observed, + "log_excerpt": logs, + } + return attachment["id"] + + +def scenario_lease_recovery(client, actor, chat, content, evidence): + compose("stop", "worker") + wait_stopped("worker") + attachment = upload(client, actor, chat, content, "grade4-lease-recovery.png") + # The one-shot worker claims the job, then exits before any side effect. + # A short lease keeps the demonstration quick without changing the default. + one_shot = compose( + "run", + "--rm", + "--no-deps", + "-e", + "WORKER_CRASH_AFTER_CLAIM=true", + "-e", + "JOB_LEASE_SECONDS=10", + "worker", + check_code=False, + timeout=120, + ) + check(one_shot.returncode == 10, f"crash-after-claim exit={one_shot.returncode}") + claimed = job_row(attachment["id"]) + check( + claimed["status"] == "processing" and claimed["processing_started_at"] is not None, + "job was not left in processing", + ) + recreate_worker({"JOB_LEASE_SECONDS": "10", "LEASE_SWEEP_SECONDS": "2"}) + ready = client.wait_ready(actor, attachment, wait_seconds=90) + observed = verify_ready_attachment(client, actor, ready, content) + row = job_row(attachment["id"]) + check(row["status"] == "ready", "lease recovery did not finish ready") + logs = relevant_logs( + worker_logs(), + [attachment["id"], row["job_id"], "recovered expired", "processing attachment"], + ) + check(any("recovered expired job leases" in line for line in logs), "sweeper log missing") + evidence["lease_recovery"] = { + "attachment_id": attachment["id"], + "one_shot_exit": one_shot.returncode, + "processing_before_expiry": True, + "attempts": row["attempt"], + "ready_after_sweep": True, + "evidence": observed, + "log_excerpt": logs, + } + restore_worker() + return attachment["id"] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--out", type=Path, default=ROOT / "evidence/lab5-grade4.json") + parser.add_argument("--markdown", type=Path, default=ROOT / "evidence/lab5-grade4.md") + parser.add_argument( + "--scenario", + choices=( + "all", + "worker-stopped", + "broker-outage", + "crash-after-effect", + "transient-retry", + "permanent-failure-replay", + "lease-recovery", + ), + default="all", + ) + args = parser.parse_args() + + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + client = Client(args.base_url) + actor, _, _, chat = client.bootstrap() + evidence = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "fixture": { + "bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "width": 320, + "height": 200, + }, + "scenarios": {}, + } + + selected = [ + ("worker-stopped", scenario_worker_stopped), + ("broker-outage", scenario_broker_outage), + ("crash-after-effect", scenario_crash_after_effect), + ("transient-retry", scenario_transient_retry), + ("permanent-failure-replay", scenario_permanent_failure_and_replay), + ("lease-recovery", scenario_lease_recovery), + ] + try: + for name, function in selected: + if args.scenario not in ("all", name): + continue + print(f"== {name}", flush=True) + function(client, actor, chat, content, evidence["scenarios"]) + print(f"== {name}: OK", flush=True) + finally: + restore_worker() + restore_api() + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(evidence, indent=2) + "\n") + + lines = [ + "# Lab5 grade-4 runtime evidence", + "", + f"Generated: `{evidence['generated_at']}`", + "", + "Synthetic fixture: 320x200 PNG, public contract fixture; no tokens or credentials are stored.", + "", + "| Scenario | Observation |", + "| --- | --- |", + ] + s = evidence["scenarios"] + if "worker_stopped" in s: + lines.append( + "| Worker stopped during upload | 202 queued, content 409, message linked while queued, " + f"ready after worker start; queue {s['worker_stopped']['queue_seconds']} s, " + f"attempt {s['worker_stopped']['row']['attempt']} |" + ) + if "broker_outage" in s: + lines.append( + "| Broker outage (COMMIT/publish gap) | 202 accepted with `published_at IS NULL`, " + "API readiness stayed 200, job published and ready after RabbitMQ restart |" + ) + if "crash_after_effect" in s: + lines.append( + "| Crash after effect, before ack | worker restart count " + f"{s['crash_after_effect']['worker_restart_count_before']} -> " + f"{s['crash_after_effect']['worker_restart_count_after']}, " + f"{s['crash_after_effect']['variant_objects']} deterministic variant objects, ready |" + ) + if "transient_retry" in s: + lines.append( + "| Transient error + jittered retry | failed once, attempt " + f"{s['transient_retry']['attempts']}, ready |" + ) + if "permanent_failure_replay" in s: + lines.append( + "| Permanent error + explicit replay | failed `invalid_image`, content/link 409, " + f"replay_count {s['permanent_failure_replay']['replay_count']}, ready with " + f"{s['permanent_failure_replay']['variant_objects_after_replay']} variant objects |" + ) + if "lease_recovery" in s: + lines.append( + "| Crash after claim / expired lease | one-shot exit 10, processing, sweeper recovered, " + f"attempt {s['lease_recovery']['attempts']}, ready |" + ) + lines.extend(["", "## Sanitized log excerpts", ""]) + for name, payload in s.items(): + excerpt = payload.get("log_excerpt") + if not excerpt: + continue + lines.append(f"### {name}") + lines.append("") + lines.append("```text") + lines.extend(line for line in excerpt[:12]) + lines.append("```") + lines.append("") + args.markdown.write_text("\n".join(lines) + "\n") + print(json.dumps({"ok": True, "out": str(args.out), "markdown": str(args.markdown)})) + + +if __name__ == "__main__": + try: + main() + except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc: + print("FAIL:", exc) + raise SystemExit(1) diff --git a/scripts/restart.sh b/scripts/restart.sh index 38c4bc1d..738cc63d 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,4 +1,6 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -docker compose --env-file .env restart postgres redis rustfs haproxy api +# Lab5: restart the broker and worker as well, with all volumes preserved. The +# persistence test then proves an already accepted job still completes. +docker compose --env-file .env restart postgres redis rustfs rabbitmq haproxy api worker diff --git a/scripts/verify-attachment-failover.sh b/scripts/verify-attachment-failover.sh index f02e2c6a..fb119f9f 100755 --- a/scripts/verify-attachment-failover.sh +++ b/scripts/verify-attachment-failover.sh @@ -1,6 +1,7 @@ #!/bin/sh -# Verify the lab4 durability claim: upload through the load balancer, stop the -# API replica that served the upload, then download the same attachment through +# Verify the lab5 durability claim: accept an upload through the load balancer +# (202 after durable staging plus outbox), let the worker make it ready, stop +# the API replica that accepted it, then download the same attachment through # another replica and compare the SHA-256. The stopped replica is restored on # exit so the stand is left as it was found. set -eu diff --git a/tests/custom/attachment_flow.py b/tests/custom/attachment_flow.py index 77866fef..63901334 100755 --- a/tests/custom/attachment_flow.py +++ b/tests/custom/attachment_flow.py @@ -29,14 +29,15 @@ def upload(args): response = c.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", - 201, + 202, actor=alice, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment", ) - attachment = response["json"] - check(attachment["status"] == "ready", "upload is not ready") + accepted_by = response["headers"].get("X-Instance-Id") + attachment = c.wait_ready(alice, response["json"], wait_seconds=60) + check(attachment["status"] == "ready", "upload did not become ready") check( attachment["sha256"] == hashlib.sha256(content).hexdigest(), "upload checksum mismatch", @@ -45,7 +46,7 @@ def upload(args): "attachment_id": attachment["id"], "sha256": attachment["sha256"], "size_bytes": attachment["size_bytes"], - "instance_id": response["headers"].get("X-Instance-Id"), + "instance_id": accepted_by, "token": alice["token"], "chat_id": chat["id"], "content_length": len(content), diff --git a/tests/custom/attachments.py b/tests/custom/attachments.py index 922d39f2..309a1a4c 100755 --- a/tests/custom/attachments.py +++ b/tests/custom/attachments.py @@ -9,8 +9,10 @@ envelope, not a 500. """ import argparse import hashlib +import struct import sys import uuid +import zlib from pathlib import Path ROOT = Path(__file__).resolve().parents[2] @@ -20,6 +22,34 @@ from client import Client, check # noqa: E402 FIXTURE = ROOT / "tests" / "fixtures" / "sample.png" MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +MAX_IMAGE_PIXELS = 40_000_000 + + +def png_chunk(kind, payload): + crc = zlib.crc32(kind + payload) & 0xFFFFFFFF + return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", crc) + + +def oversized_png(width, height): + """Build a valid grayscale PNG whose dimensions exceed the pixel cap. + + The image is solid black, so the IDAT stream stays tiny and the request is + far below the byte limit. Only the declared width*height can trigger 413. + """ + ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0) + compressor = zlib.compressobj(9) + idat = bytearray() + row = b"\x00" + b"\x00" * width # filter type 0 + one 8-bit sample per pixel + for _ in range(height): + idat += compressor.compress(row) + idat += compressor.flush() + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk(b"IHDR", ihdr) + + png_chunk(b"IDAT", bytes(idat)) + + png_chunk(b"IEND", b"") + ) + def upload(client, actor, chat_id, raw, filename, content_type): @@ -27,7 +57,7 @@ def upload(client, actor, chat_id, raw, filename, content_type): return client.call( "POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", - expected=(201, 400, 404, 413, 415), + expected=(202, 400, 404, 413, 415), actor=actor, raw=raw, headers=headers, @@ -40,20 +70,31 @@ def main(): parser.add_argument("--ca-file", default=None) args = parser.parse_args() - c = Client(args.base_url, args.ca_file) + c = Client(args.base_url, args.ca_file, timeout=60) alice, bob, eve, chat = c.bootstrap() png = FIXTURE.read_bytes() # Happy path first, so the negative checks can refer to a real attachment. - good = c.attachment(alice, chat)[0] + good = c.wait_ready(alice, c.attachment(alice, chat)[0]) check(good["status"] == "ready", "valid upload is not ready") check(good["error_code"] is None, "ready attachment carries an error") + check( + {item["name"] for item in good["variants"]} == {"thumbnail", "crop", "optimized"}, + "ready attachment is missing a grade-4 variant", + ) # 1. One byte over the contract limit must be a 413, decided before decode. oversized = b"\x00" * (MAX_UPLOAD_BYTES + 1) response = upload(c, alice, chat["id"], oversized, "big.png", "image/png") check(response["status"] == 413, "oversized upload did not return 413") + # 1b. A decompression bomb passes the byte limit but exceeds the pixel + # cap. It must be rejected from the header before pixel allocation. + bomb = oversized_png(7000, 6000) # 42 Mp > 40 Mp hard cap + check(len(bomb) < MAX_UPLOAD_BYTES, "pixel-bomb fixture unexpectedly exceeds bytes") + response = upload(c, alice, chat["id"], bomb, "bomb.png", "image/png") + check(response["status"] == 413, "pixel-limit upload did not return 413") + # 2. A file with a valid PNG signature but a broken body must be a 415. corrupt = png[:8] + b"\x00" * 64 response = upload(c, alice, chat["id"], corrupt, "broken.png", "image/png") @@ -95,7 +136,7 @@ def main(): body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]}, schema="Chat", )["json"] - other_attachment = c.attachment(alice, other)[0] + other_attachment = c.wait_ready(alice, c.attachment(alice, other)[0]) c.call( "POST", f"/api/v1/chats/{chat['id']}/messages", @@ -159,7 +200,7 @@ def main(): ) print( - "PASS: graded negative cases (413/415/404/400), no 5xx, " + "PASS: graded negative cases (413 byte/pixel, 415/404/400), no 5xx, " "cross-member download checksum matches" ) diff --git a/tests/custom/concurrent_uploads.py b/tests/custom/concurrent_uploads.py index 59e0da67..6f3e341b 100644 --- a/tests/custom/concurrent_uploads.py +++ b/tests/custom/concurrent_uploads.py @@ -72,7 +72,7 @@ def main(): response = client.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=concurrent-{index}.png", - 201, + 202, actor=alice, raw=png, headers={"Content-Type": "image/png"}, @@ -102,8 +102,11 @@ def main(): instances = sorted({item["instance"] for item in results if item["instance"]}) check(bool(instances), "no X-Instance-Id was observed") - # Read one uploaded object back and compare the exact bytes before reporting - # success; ten 201 responses alone are not enough. + # Lab5: a 202 only means the intent is durable. Wait for the worker, then + # read one object back and compare the exact bytes; accepted responses + # alone are not enough. + for item in results: + c.wait_ready(alice, item, wait_seconds=120) download = c.call( "GET", f"/api/v1/attachments/{results[0]['id']}/content", actor=alice )