diff --git a/.env.example b/.env.example index 119384b5..c13c3ba8 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,54 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Copy to .env for a local synthetic environment. Never commit .env. APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 + +DATABASE_NAME=messenger +DATABASE_USER=messenger +DATABASE_PASSWORD=replace-with-a-long-random-database-password + +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=replace-with-a-long-random-rabbitmq-password +RABBITMQ_VHOST=/ +ATTACHMENT_JOB_EXCHANGE=attachments.v1 +ATTACHMENT_JOB_QUEUE=attachments.process.level4.v1 +ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1 +ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1 +ATTACHMENT_DLQ_QUEUE=attachments.failed.v1 +ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1 +WORKER_CONCURRENCY=2 +WORKER_PREFETCH=1 +JOB_MAX_ATTEMPTS=5 +JOB_RETRY_BASE_DELAY=1s +JOB_RETRY_MAX_DELAY=30s +JOB_TIMEOUT=60s +JOB_LEASE_DURATION=90s +JOB_LEASE_REAPER_INTERVAL=5s +WORKER_MAX_IMAGE_PIXELS=40000000 +WORKER_MEMORY_BUDGET_BYTES=536870912 +WORKER_MEMORY_WAIT=2s + +# Fault injection is opt-in and intended only for an isolated test stand. +FAULT_CRASH_AFTER_OUTBOX_COMMIT=false +FAULT_CRASH_AFTER_RESULT_COMMIT=false + S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=replace-with-a-long-random-api-secret +MINIO_ROOT_USER=minio-root +MINIO_ROOT_PASSWORD=replace-with-a-different-long-random-root-secret + +SESSION_TTL_SECONDS=3600 +IDEMPOTENCY_TTL_SECONDS=86400 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 +MAX_CONCURRENT_UPLOADS=3 +UPLOAD_PERMIT_WAIT_MILLIS=10000 + +# Staging objects are deleted only after the final result is committed. +# A separate lifecycle/cleanup policy may remove abandoned staging objects +# after this retention interval; it must never remove active jobs. STAGING_RETENTION_SECONDS=86400 +ATTACHMENT_CLEANUP_GRACE_SECONDS=3600 +ATTACHMENT_CLEANUP_DRY_RUN=false +ATTACHMENT_RECONCILE_INTERVAL_MILLIS=600000 +ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS=60000 diff --git a/.gitignore b/.gitignore index b5a71896..affc5505 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ backups/ # Runtime/build artifacts .venv/ +.gradle/ __pycache__/ *.py[cod] node_modules/ diff --git a/REPORT.md b/REPORT.md index e98f75e9..687415e4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,71 @@ # Отчёт — лабораторная 5 - Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Целевая оценка: 4 - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Библиотеки: Spring AMQP, RabbitMQ Java client, Thumbnailator, webp-imageio, MinIO SDK, PostgreSQL/Flyway +- Помощь ИИ: проектирование pipeline, реализация и подготовка тестов; результат проверен сборкой, unit- и black-box-тестами ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop/Engine с Compose v2, Java 23 и Python 3.9+. -## Архитектура и решения +```sh +cp .env.example .env +# Заменить все replace-with-* уникальными локальными значениями. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +python -X utf8 tests/smoke.py --base-url http://localhost:8080 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных выполняется только явной командой оператора с `--volumes`. -## Доказательства критериев +## Pipeline и гарантии -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет неизменённый original в приватный `staging//original`, затем одной транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`. Запись job является transactional outbox. Фоновый publisher публикует persistent JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а HTTP `202` означает, что staging и намерение обработки уже устойчивы. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Отдельный `worker` захватывает job с lease, читает доверенный staging key из PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только `schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют. Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже `ready` job — no-op. -## Эксперименты и отказы +Обязательные варианты pipeline version 1: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Вариант | Правило | Формат/качество | +| --- | --- | --- | +| `original` | bytes и SHA-256 совпадают с upload | исходный MIME | +| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality 0.85 | +| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality 0.88 | +| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality 0.82 | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных metadata. -## Ограничения и следующие шаги +## Ограничения worker и retry -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Upload: `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000`. +- Worker повторно проверяет header/dimensions до полного decode и применяет тот же pixel cap. +- Оценка памяти на job резервируется общим честным semaphore; бюджет `WORKER_MEMORY_BUDGET_BYTES=536870912`, ожидание `WORKER_MEMORY_WAIT=2s`. Compose дополнительно ограничивает worker `1024m`, JVM использует до 70% container RAM. +- Обработка ограничена `JOB_TIMEOUT=60s`; просроченный task теряет lease и не может зафиксировать результат. +- `WORKER_CONCURRENCY=2`, `WORKER_PREFETCH=1`; lease 90s, reaper каждые 5s. +- Не более `JOB_MAX_ATTEMPTS=5`. Временные ошибки получают exponential backoff с jitter 50–100% и cap 30s. Ошибки decode/format/size постоянные и сразу становятся безопасным `failed/error_code`; exhausted retry — `retry_exhausted`. +- Permanent malformed broker delivery и исчерпанная работа dead-lettered в durable `attachments.failed.v1`. Реестр failed остаётся в PostgreSQL. + +Операторский replay выполняется `scripts/replay-failed.ps1 -JobId ` или `scripts/replay-failed.sh `. Одна транзакция блокирует одновременно failed job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние. Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`. + +После безопасного COMMIT результата worker best-effort удаляет staging. `StagingCleaner` удаляет только неактивные объекты старше `STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён. + +## Fault injection и проверки + +| Критерий | Команда | Результат | +| --- | --- | --- | +| Unit tests | `gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' --tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'` | `BUILD SUCCESSFUL` | +| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:8080` | PASS | +| Worker stop, broker/API restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows | +| COMMIT→publish и result COMMIT→ack crash | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи автоматически дошли до ready | +| DLQ, replay, lease | `python -X utf8 student_tests/lab5_dlq_replay_lease.py` | PASS; DLQ +1, одна audit-запись, ready не replayed, expired lease восстановлен | + +Fault flags выключены по умолчанию. `FAULT_CRASH_AFTER_OUTBOX_COMMIT=true` один раз завершает API через `Runtime.halt(87)` после DB COMMIT до publish. После restart republisher находит `published=false`, и job достигает ready. `FAULT_CRASH_AFTER_RESULT_COMMIT=true` один раз завершает worker через `Runtime.halt(86)` после атомарного ready до consumer ack; RabbitMQ redelivery проверяет ready и завершает ту же job без второго набора вариантов. + +Подробный журнал: `evidence/lab5-level4-2026-11-19.md`. + +## Ограничения + +Это уровень 4, не исследовательский трек 5: производительность для матрицы concurrency/prefetch не измерялась, multi-version reorder не исследовался. RabbitMQ, PostgreSQL и MinIO развёрнуты по одному экземпляру; volumes переживают container restart, но не потерю host/disk. At-least-once + идемпотентный эффект не является exactly-once доставкой. diff --git a/build.gradle b/build.gradle index 22435bce..5c37fd68 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,7 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-data-redis' + implementation 'org.springframework.boot:spring-boot-starter-amqp' implementation 'org.springframework.boot:spring-boot-flyway' implementation 'org.flywaydb:flyway-core' implementation 'org.flywaydb:flyway-database-postgresql' @@ -31,6 +32,8 @@ dependencies { implementation 'org.bouncycastle:bcprov-jdk18on:1.84' implementation 'org.apache.commons:commons-pool2' implementation 'io.minio:minio:8.6.0' + implementation 'net.coobird:thumbnailator:0.4.20' + implementation 'org.sejda.imageio:webp-imageio:0.1.6' runtimeOnly 'org.postgresql:postgresql' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' diff --git a/compose.yaml b/compose.yaml index bc0ebf42..82fa8d81 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,4 @@ -name: hla-lab4 +name: hla-lab5 x-api-environment: &api-environment APP_HOST: 0.0.0.0 @@ -30,6 +30,29 @@ x-api-environment: &api-environment MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3} UPLOAD_PERMIT_WAIT_MILLIS: ${UPLOAD_PERMIT_WAIT_MILLIS:-10000} + RABBITMQ_HOST: rabbitmq + RABBITMQ_PORT: 5672 + RABBITMQ_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + RABBITMQ_VHOST: ${RABBITMQ_VHOST:-/} + ATTACHMENT_JOB_EXCHANGE: ${ATTACHMENT_JOB_EXCHANGE:-attachments.v1} + ATTACHMENT_JOB_QUEUE: ${ATTACHMENT_JOB_QUEUE:-attachments.process.level4.v1} + ATTACHMENT_JOB_ROUTING_KEY: ${ATTACHMENT_JOB_ROUTING_KEY:-attachment.process.v1} + ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1} + ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1} + ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_RETRY_BASE_DELAY: ${JOB_RETRY_BASE_DELAY:-1s} + JOB_RETRY_MAX_DELAY: ${JOB_RETRY_MAX_DELAY:-30s} + JOB_TIMEOUT: ${JOB_TIMEOUT:-60s} + JOB_LEASE_DURATION: ${JOB_LEASE_DURATION:-90s} + JOB_LEASE_REAPER_INTERVAL: ${JOB_LEASE_REAPER_INTERVAL:-5s} + WORKER_MAX_IMAGE_PIXELS: ${WORKER_MAX_IMAGE_PIXELS:-40000000} + WORKER_MEMORY_BUDGET_BYTES: ${WORKER_MEMORY_BUDGET_BYTES:-536870912} + WORKER_MEMORY_WAIT: ${WORKER_MEMORY_WAIT:-2s} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600} ATTACHMENT_CLEANUP_DRY_RUN: ${ATTACHMENT_CLEANUP_DRY_RUN:-false} ATTACHMENT_RECONCILE_INTERVAL_MILLIS: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:-600000} @@ -69,6 +92,26 @@ services: mem_limit: 256m networks: [backend] + rabbitmq: + image: rabbitmq:4.1.4-management-alpine + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/} + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2" + volumes: + - rabbitmq-data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 10s + start_period: 60s + retries: 12 + restart: unless-stopped + cpus: 0.5 + mem_limit: 1024m + networks: [backend] + minio: image: bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd0e172c4cc0870f4bdc3142018e2a37be9acf04d68f386600daad427e0cab environment: @@ -103,13 +146,24 @@ services: command: - | set -eu - mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" - mc mb --ignore-existing "local/$$S3_BUCKET" - mc anonymous set none "local/$$S3_BUCKET" sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy/minio-api-policy.json > /tmp/minio-api-policy.json - mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" || true - mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json || true - mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID" + attempt=0 + until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" \ + && mc mb --ignore-existing "local/$$S3_BUCKET" \ + && mc anonymous set none "local/$$S3_BUCKET" \ + && (mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" \ + || mc admin user info local "$$S3_ACCESS_KEY_ID") \ + && (mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json \ + || mc admin policy info local messenger-attachments-rw) \ + && mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID" + do + attempt=$$((attempt + 1)) + if [ "$$attempt" -ge 30 ]; then + echo "MinIO bootstrap did not complete after 30 attempts" >&2 + exit 1 + fi + sleep 2 + done restart: "no" networks: [backend] @@ -144,38 +198,45 @@ services: # One-shot Flyway runner. It is invoked explicitly before API replicas. migrate: - image: hla-lab4-api:local + image: hla-lab5-app:local build: context: . environment: <<: *api-environment FLYWAY_ENABLED: "true" MIGRATE_ONLY: "true" + MESSENGER_ROLE: migration SPRING_MAIN_WEB_APPLICATION_TYPE: none depends_on: postgres: condition: service_healthy redis: condition: service_healthy + rabbitmq: + condition: service_healthy restart: "no" cpus: 0.75 mem_limit: 768m networks: [backend] api: - image: hla-lab4-api:local + image: hla-lab5-app:local build: context: . environment: <<: *api-environment FLYWAY_ENABLED: "false" + MESSENGER_ROLE: api + FAULT_CRASH_AFTER_OUTBOX_COMMIT: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:-false} depends_on: - postgres: - condition: service_healthy + migrate: + condition: service_completed_successfully redis: condition: service_healthy minio-init: condition: service_completed_successfully + rabbitmq: + condition: service_healthy healthcheck: test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"] interval: 2s @@ -189,6 +250,31 @@ services: mem_limit: 768m networks: [backend] + worker: + image: hla-lab5-app:local + environment: + <<: *api-environment + FLYWAY_ENABLED: "false" + MESSENGER_ROLE: worker + SPRING_MAIN_WEB_APPLICATION_TYPE: none + JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70 + FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false} + depends_on: + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + healthcheck: + disable: true + init: true + stop_grace_period: 20s + restart: unless-stopped + cpus: 1.0 + mem_limit: 1024m + networks: [backend] + nginx: image: nginx:1.27.4-bookworm ports: @@ -217,3 +303,4 @@ volumes: postgres-data: redis-data: minio-data: + rabbitmq-data: diff --git a/evidence/lab5-2026-11-18.md b/evidence/lab5-2026-11-18.md new file mode 100644 index 00000000..a782c2fc --- /dev/null +++ b/evidence/lab5-2026-11-18.md @@ -0,0 +1,41 @@ +# Lab5 verification — 2026-11-18 + +Среда: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23, Spring Boot 4.1.1, RabbitMQ 4.1.4. + +## Статические и unit-проверки + +```text +docker compose --env-file .env.example config --quiet +PASS (exit 0) + +gradlew test --tests '*AttachmentServiceTest' --tests '*ThumbnailProcessorTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest' +BUILD SUCCESSFUL +``` + +## Публичный сценарий + +```text +python tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +``` + +В момент проверки отдельные services `api`, `worker`, `rabbitmq`, `postgres`, `redis`, `minio` и `nginx` были запущены; RabbitMQ queue `attachments.process.v1` имела `durable=true`. + +## Остановка worker, restart и redelivery + +```text +python student_tests/lab5_worker_recovery.py +PASS: stopped worker, durable restart recovery and duplicate delivery +``` + +Сценарий теста: + +1. останавливает worker; +2. получает upload `202 queued`, проверяет `409` content и привязку к сообщению; +3. перезапускает API и RabbitMQ без удаления volumes; +4. запускает worker и ждёт `ready`; +5. сверяет original bytes и единственный thumbnail; +6. публикует тот же `job_id` повторно; +7. проверяет terminal `ready` и ровно одну запись thumbnail. + +Синтетические пароли и токены тест не печатает и в evidence не сохраняет. diff --git a/evidence/lab5-level4-2026-11-19.md b/evidence/lab5-level4-2026-11-19.md new file mode 100644 index 00000000..00b9be60 --- /dev/null +++ b/evidence/lab5-level4-2026-11-19.md @@ -0,0 +1,56 @@ +# Lab5 level 4 verification — 2026-11-19 + +Среда: Windows, Docker Compose, Java 23, Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, MinIO. + +## Статические и unit-проверки + +```text +python -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +docker compose --env-file .env.example config --quiet +exit 0 + +gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' \ + --tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' \ + --tests '*StagingCleanerTest' +BUILD SUCCESSFUL +``` + +`ImageVariantProcessorTest` проверяет реальные JPEG/WebP signatures, ImageIO decode и размеры thumbnail/crop/optimized. `RetryPolicyTest` проверяет bounded exponential backoff и jitter. + +## API и recovery + +```text +python -X utf8 tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +python -X utf8 student_tests/lab5_worker_recovery.py +PASS: stopped worker, durable restart recovery and duplicate delivery +``` + +Recovery test останавливает worker, получает `202 queued`, перезапускает API и RabbitMQ без удаления volumes, запускает worker и сверяет original SHA-256, thumbnail 256×160, crop 128×128, WebP signature. Повторная публикация того же `job_id` оставляет `ready` и ровно три variant rows. + +## Доказанные окна отказа + +```text +python -X utf8 student_tests/lab5_fault_injection.py +PASS: outbox commit/publish crash and result commit/ack crash recovered +``` + +1. API завершается после COMMIT attachment/outbox до publish. Перезапущенный API republisher публикует сохранённое намерение; новый upload не создаётся вручную. +2. Worker завершается после записи объектов и atomic ready COMMIT до ack. Redelivery завершает тот же effect; variant rows остаются уникальными. + +## DLQ, replay и lease + +```text +python -X utf8 student_tests/lab5_dlq_replay_lease.py +PASS: DLQ, guarded replay and expired lease recovery +``` + +- malformed persistent JSON был permanent failure и увеличил `attachments.failed.v1` ровно на 1; +- failed job replayed один раз, audit count = 1; повтор в queued/ready дал no-op; +- после replay получены те же три variant names без второго набора; +- искусственно истёкший processing lease был requeued, attempt увеличился, job дошёл до ready. + +Секреты, session tokens и contents сообщений в evidence не сохранялись. Docker volumes в проверках не удалялись. diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json index 97085216..0ee222d4 100644 --- a/infra/minio-api-policy.json +++ b/infra/minio-api-policy.json @@ -19,7 +19,8 @@ "s3:DeleteObject" ], "Resource": [ - "arn:aws:s3:::__S3_BUCKET__/attachments/*" + "arn:aws:s3:::__S3_BUCKET__/attachments/*", + "arn:aws:s3:::__S3_BUCKET__/staging/*" ] } ] diff --git a/scripts/replay-failed.ps1 b/scripts/replay-failed.ps1 new file mode 100644 index 00000000..4bd2343e --- /dev/null +++ b/scripts/replay-failed.ps1 @@ -0,0 +1,57 @@ +param( + [Parameter(Mandatory = $true)] + [string]$JobId, + [string]$RequestedBy = $env:USERNAME, + [string]$EnvFile = ".env" +) + +$parsedJobId = [guid]::Empty +if (-not [guid]::TryParse($JobId, [ref]$parsedJobId)) { + throw "JobId must be a UUID" +} +if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') { + throw "RequestedBy contains unsupported characters" +} + +$sql = @" +WITH candidate AS ( + SELECT j.id, j.attachment_id, j.attempt + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = '$parsedJobId'::uuid + AND j.status = 'failed' AND a.status = 'failed' + FOR UPDATE OF j, a +), audited AS ( + INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt) + SELECT id, '$RequestedBy', attempt FROM candidate + RETURNING job_id +), reset_job AS ( + UPDATE attachment_jobs j + SET status = 'queued', attempt = 1, published = false, + last_error_code = NULL, next_attempt_at = clock_timestamp(), + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c, audited a + WHERE j.id = c.id AND a.job_id = c.id + RETURNING j.attachment_id +), reset_attachment AS ( + UPDATE attachments a + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + FROM reset_job r + WHERE a.id = r.attachment_id AND a.status = 'failed' + RETURNING a.id +) +SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment) + THEN 'replayed' ELSE 'no-op' END; +"@ + +$result = docker compose --env-file $EnvFile exec -T postgres ` + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql +if ($LASTEXITCODE -ne 0) { + throw "Replay transaction failed" +} + +if (($result | Select-Object -Last 1).Trim() -eq "replayed") { + Write-Output "Replay requested for failed job $parsedJobId by $RequestedBy" +} else { + Write-Output "No-op: job $parsedJobId is missing, not failed, or already recovered" +} diff --git a/scripts/replay-failed.sh b/scripts/replay-failed.sh new file mode 100644 index 00000000..25fa5a1c --- /dev/null +++ b/scripts/replay-failed.sh @@ -0,0 +1,45 @@ +#!/bin/sh +set -eu + +job_id="${1:?usage: replay-failed.sh JOB_UUID [REQUESTED_BY] [ENV_FILE]}" +requested_by="${2:-operator}" +env_file="${3:-.env}" + +case "$job_id" in + ????????-????-????-????-????????????) ;; + *) echo "JobId must be a UUID" >&2; exit 2 ;; +esac +case "$requested_by" in + *[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;; +esac + +sql="WITH candidate AS ( + SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed' + FOR UPDATE OF j, a +), audited AS ( + INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt) + SELECT id, '$requested_by', attempt FROM candidate RETURNING job_id +), reset_job AS ( + UPDATE attachment_jobs j SET status = 'queued', attempt = 1, published = false, + last_error_code = NULL, next_attempt_at = clock_timestamp(), + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c, audited a WHERE j.id = c.id AND a.job_id = c.id + RETURNING j.attachment_id +), reset_attachment AS ( + UPDATE attachments a SET status = 'queued', error_code = NULL, + updated_at = clock_timestamp() + FROM reset_job r WHERE a.id = r.attachment_id AND a.status = 'failed' + RETURNING a.id +) +SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment) + THEN 'replayed' ELSE 'no-op' END;" + +result="$(docker compose --env-file "$env_file" exec -T postgres \ + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")" +if [ "$result" = "replayed" ]; then + echo "Replay requested for failed job $job_id by $requested_by" +else + echo "No-op: job $job_id is missing, not failed, or already recovered" +fi diff --git a/src/main/java/com/example/mtusi_messenger/config/RabbitJobConfig.java b/src/main/java/com/example/mtusi_messenger/config/RabbitJobConfig.java new file mode 100644 index 00000000..ec334a91 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/RabbitJobConfig.java @@ -0,0 +1,57 @@ +package com.example.mtusi_messenger.config; + +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.DirectExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RabbitJobConfig { + @Bean + DirectExchange attachmentExchange(@Value("${messenger.jobs.exchange}") String name) { + return new DirectExchange(name, true, false); + } + + @Bean + Queue attachmentQueue( + @Value("${messenger.jobs.queue}") String name, + @Value("${messenger.jobs.dead-letter-exchange}") String deadLetterExchange, + @Value("${messenger.jobs.dead-letter-routing-key}") String deadLetterRoutingKey) { + return QueueBuilder.durable(name) + .withArgument("x-dead-letter-exchange", deadLetterExchange) + .withArgument("x-dead-letter-routing-key", deadLetterRoutingKey) + .build(); + } + + @Bean + DirectExchange attachmentDeadLetterExchange( + @Value("${messenger.jobs.dead-letter-exchange}") String name) { + return new DirectExchange(name, true, false); + } + + @Bean + Queue attachmentDeadLetterQueue(@Value("${messenger.jobs.dead-letter-queue}") String name) { + return QueueBuilder.durable(name).build(); + } + + @Bean + Binding attachmentBinding( + @Qualifier("attachmentQueue") Queue attachmentQueue, + @Qualifier("attachmentExchange") DirectExchange attachmentExchange, + @Value("${messenger.jobs.routing-key}") String routingKey) { + return BindingBuilder.bind(attachmentQueue).to(attachmentExchange).with(routingKey); + } + + @Bean + Binding attachmentDeadLetterBinding( + @Qualifier("attachmentDeadLetterQueue") Queue queue, + @Qualifier("attachmentDeadLetterExchange") DirectExchange exchange, + @Value("${messenger.jobs.dead-letter-routing-key}") String routingKey) { + return BindingBuilder.bind(queue).to(exchange).with(routingKey); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java b/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java index 9d5a116b..72350f53 100644 --- a/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java +++ b/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java @@ -1,7 +1,6 @@ package com.example.mtusi_messenger.controller; import com.example.mtusi_messenger.dto.AttachmentResponse; -import com.example.mtusi_messenger.exception.ApiExceptions; import com.example.mtusi_messenger.model.Attachment; import com.example.mtusi_messenger.security.SessionPrincipal; import com.example.mtusi_messenger.service.AttachmentService; @@ -47,7 +46,7 @@ public class AttachmentController { ) throws IOException { Attachment attachment = attachments.upload(parse(chatId), principal.user().id(), filename, contentType, request.getContentLengthLong(), request.getInputStream()); - return ResponseEntity.status(HttpStatus.CREATED).body(AttachmentResponse.from(attachment)); + return ResponseEntity.status(HttpStatus.ACCEPTED).body(AttachmentResponse.from(attachment)); } @GetMapping("/attachments/{attachmentId}") @@ -64,15 +63,13 @@ public class AttachmentController { @PathVariable String attachmentId, @RequestParam(defaultValue = "original") String variant ) { - if (!"original".equals(variant)) { - throw ApiExceptions.invalid("Unsupported attachment variant"); - } - AttachmentService.Download download = attachments.download(parse(attachmentId), principal.user().id()); + AttachmentService.Download download = attachments.download( + parse(attachmentId), principal.user().id(), variant); return ResponseEntity.ok() - .contentType(MediaType.parseMediaType(download.attachment().contentType())) + .contentType(MediaType.parseMediaType(download.contentType())) .contentLength(download.bytes().length) .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.inline() - .filename(download.attachment().filename(), StandardCharsets.UTF_8).build().toString()) + .filename(download.filename(), StandardCharsets.UTF_8).build().toString()) .body(download.bytes()); } } diff --git a/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java b/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java index 01a551b0..7d7d2fa6 100644 --- a/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java +++ b/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java @@ -14,7 +14,7 @@ public record AttachmentResponse(UUID id, long size_bytes, String sha256, String status, - List variants, + List variants, String error_code, Instant created_at, Instant updated_at) { @@ -27,9 +27,9 @@ public record AttachmentResponse(UUID id, attachment.contentType(), attachment.sizeBytes(), attachment.sha256(), - "ready", - List.of(), - null, + attachment.status(), + attachment.variants().stream().map(AttachmentVariantResponse::from).toList(), + attachment.errorCode(), attachment.createdAt(), attachment.updatedAt()); } diff --git a/src/main/java/com/example/mtusi_messenger/dto/AttachmentVariantResponse.java b/src/main/java/com/example/mtusi_messenger/dto/AttachmentVariantResponse.java new file mode 100644 index 00000000..7bbdc3c1 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/AttachmentVariantResponse.java @@ -0,0 +1,16 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.AttachmentVariant; + +public record AttachmentVariantResponse(String name, + String content_type, + long size_bytes, + int width, + int height, + String sha256) { + public static AttachmentVariantResponse from(AttachmentVariant variant) { + return new AttachmentVariantResponse( + variant.name(), variant.contentType(), variant.sizeBytes(), + variant.width(), variant.height(), variant.sha256()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/Attachment.java b/src/main/java/com/example/mtusi_messenger/model/Attachment.java index b93415be..acb2515c 100644 --- a/src/main/java/com/example/mtusi_messenger/model/Attachment.java +++ b/src/main/java/com/example/mtusi_messenger/model/Attachment.java @@ -1,6 +1,7 @@ package com.example.mtusi_messenger.model; import java.time.Instant; +import java.util.List; import java.util.UUID; public record Attachment(UUID id, @@ -11,6 +12,27 @@ public record Attachment(UUID id, long sizeBytes, String sha256, String objectKey, + String stagingKey, + String status, + List variants, + String errorCode, Instant createdAt, Instant updatedAt) { + public Attachment { + variants = variants == null ? List.of() : List.copyOf(variants); + } + + public Attachment(UUID id, + UUID chatId, + UUID ownerId, + String filename, + String contentType, + long sizeBytes, + String sha256, + String objectKey, + Instant createdAt, + Instant updatedAt) { + this(id, chatId, ownerId, filename, contentType, sizeBytes, sha256, objectKey, + null, "ready", List.of(), null, createdAt, updatedAt); + } } diff --git a/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java b/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java new file mode 100644 index 00000000..0708e84e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java @@ -0,0 +1,37 @@ +package com.example.mtusi_messenger.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.UUID; + +public record AttachmentJob( + @JsonProperty("schema_version") int schemaVersion, + @JsonProperty("job_id") UUID jobId, + String type, + @JsonProperty("attachment_id") UUID attachmentId, + @JsonProperty("pipeline_version") int pipelineVersion, + int attempt, + @JsonProperty("created_at") Instant createdAt) { + + public static final int SCHEMA_VERSION = 1; + public static final int PIPELINE_VERSION = 1; + public static final String TYPE = "attachment.process.v1"; + + public AttachmentJob { + if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type) + || pipelineVersion < 1 || attempt < 1) { + throw new IllegalArgumentException("Unsupported attachment job message"); + } + } + + public static AttachmentJob first(UUID jobId, UUID attachmentId, Instant createdAt) { + return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId, + PIPELINE_VERSION, 1, createdAt); + } + + public AttachmentJob nextAttempt(int nextAttempt) { + return new AttachmentJob(schemaVersion, jobId, type, attachmentId, + pipelineVersion, nextAttempt, createdAt); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/AttachmentVariant.java b/src/main/java/com/example/mtusi_messenger/model/AttachmentVariant.java new file mode 100644 index 00000000..ff4e8092 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/AttachmentVariant.java @@ -0,0 +1,10 @@ +package com.example.mtusi_messenger.model; + +public record AttachmentVariant(String name, + String contentType, + long sizeBytes, + int width, + int height, + String sha256, + String objectKey) { +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/AttachmentJobPublisher.java b/src/main/java/com/example/mtusi_messenger/queue/AttachmentJobPublisher.java new file mode 100644 index 00000000..ce87b427 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/AttachmentJobPublisher.java @@ -0,0 +1,65 @@ +package com.example.mtusi_messenger.queue; + +import com.example.mtusi_messenger.model.AttachmentJob; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.Duration; + +@Component +public class AttachmentJobPublisher { + private final RabbitTemplate rabbit; + private final ObjectMapper mapper; + private final String exchange; + private final String routingKey; + private final long confirmTimeoutMillis; + + public AttachmentJobPublisher( + RabbitTemplate rabbit, + ObjectMapper mapper, + @Value("${messenger.jobs.exchange}") String exchange, + @Value("${messenger.jobs.routing-key}") String routingKey, + @Value("${messenger.jobs.publish-confirm-timeout:5s}") Duration confirmTimeout) { + this.rabbit = rabbit; + this.mapper = mapper; + this.exchange = exchange; + this.routingKey = routingKey; + this.confirmTimeoutMillis = confirmTimeout.toMillis(); + this.rabbit.setMandatory(true); + } + + public void publish(AttachmentJob job) { + try { + byte[] body = mapper.writeValueAsBytes(job); + MessageProperties properties = new MessageProperties(); + properties.setContentType(MessageProperties.CONTENT_TYPE_JSON); + properties.setContentEncoding("UTF-8"); + properties.setDeliveryMode(MessageDeliveryMode.PERSISTENT); + properties.setMessageId(job.jobId().toString()); + properties.setHeader("schema_version", job.schemaVersion()); + Message message = new Message(body, properties); + + rabbit.invoke(operations -> { + operations.send(exchange, routingKey, message); + operations.waitForConfirmsOrDie(confirmTimeoutMillis); + return null; + }); + } catch (JsonProcessingException error) { + throw new JobPublishException("Cannot encode attachment job", error); + } catch (RuntimeException error) { + throw new JobPublishException("RabbitMQ did not confirm attachment job", error); + } + } + + public static final class JobPublishException extends RuntimeException { + public JobPublishException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java b/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java new file mode 100644 index 00000000..7672b84f --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java @@ -0,0 +1,55 @@ +package com.example.mtusi_messenger.queue; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.Clock; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class ExpiredLeaseReaper { + private static final Logger log = LoggerFactory.getLogger(ExpiredLeaseReaper.class); + private final AttachmentRepository attachments; + private final RetryPolicy retryPolicy; + private final TransactionTemplate transaction; + private final int maxAttempts; + private final Clock clock = Clock.systemUTC(); + + public ExpiredLeaseReaper( + AttachmentRepository attachments, + RetryPolicy retryPolicy, + TransactionTemplate transaction, + @Value("${messenger.jobs.max-attempts:5}") int maxAttempts) { + this.attachments = attachments; + this.retryPolicy = retryPolicy; + this.transaction = transaction; + this.maxAttempts = maxAttempts; + } + + @Scheduled(fixedDelayString = "${messenger.jobs.lease-reaper-interval:5s}") + public void recoverExpired() { + for (var job : attachments.findExpiredJobs(50)) { + try { + transaction.executeWithoutResult(status -> { + if (job.attempt() >= maxAttempts) { + attachments.failExpired(job.jobId(), job.attachmentId(), "retry_exhausted"); + } else { + int nextAttempt = job.attempt() + 1; + attachments.requeueExpired( + job.jobId(), job.attachmentId(), nextAttempt, "processing_timeout", + clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt))); + } + }); + } catch (RuntimeException error) { + log.warn("Expired attachment lease recovery postponed for job {}", job.jobId()); + return; + } + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/RetryPolicy.java b/src/main/java/com/example/mtusi_messenger/queue/RetryPolicy.java new file mode 100644 index 00000000..306375e2 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/RetryPolicy.java @@ -0,0 +1,39 @@ +package com.example.mtusi_messenger.queue; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.concurrent.ThreadLocalRandom; + +@Component +public class RetryPolicy { + private final long baseDelayMillis; + private final long maxDelayMillis; + + public RetryPolicy( + @Value("${messenger.jobs.retry-base-delay:1s}") Duration baseDelay, + @Value("${messenger.jobs.retry-max-delay:30s}") Duration maxDelay) { + if (baseDelay.isNegative() || baseDelay.isZero() || maxDelay.compareTo(baseDelay) < 0) { + throw new IllegalArgumentException("Invalid retry delay configuration"); + } + this.baseDelayMillis = baseDelay.toMillis(); + this.maxDelayMillis = maxDelay.toMillis(); + } + + public Duration delayForAttempt(int nextAttempt) { + int exponent = Math.max(0, Math.min(30, nextAttempt - 2)); + long exponential; + try { + exponential = Math.multiplyExact(baseDelayMillis, 1L << exponent); + } catch (ArithmeticException overflow) { + exponential = maxDelayMillis; + } + long ceiling = Math.min(maxDelayMillis, exponential); + long floor = Math.max(1, ceiling / 2); + long jittered = floor == ceiling + ? ceiling + : ThreadLocalRandom.current().nextLong(floor, ceiling + 1); + return Duration.ofMillis(jittered); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/UnpublishedJobRepublisher.java b/src/main/java/com/example/mtusi_messenger/queue/UnpublishedJobRepublisher.java new file mode 100644 index 00000000..b83b1f51 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/UnpublishedJobRepublisher.java @@ -0,0 +1,34 @@ +package com.example.mtusi_messenger.queue; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class UnpublishedJobRepublisher { + private static final Logger log = LoggerFactory.getLogger(UnpublishedJobRepublisher.class); + private final AttachmentRepository attachments; + private final AttachmentJobPublisher publisher; + + public UnpublishedJobRepublisher(AttachmentRepository attachments, AttachmentJobPublisher publisher) { + this.attachments = attachments; + this.publisher = publisher; + } + + @Scheduled(fixedDelayString = "${messenger.jobs.republish-interval:5s}") + public void publishPending() { + for (var job : attachments.findUnpublishedJobs(50)) { + try { + publisher.publish(job); + attachments.markPublished(job.jobId()); + } catch (RuntimeException error) { + log.warn("Attachment job publication postponed for job {}", job.jobId()); + return; + } + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java index a6b1ce32..0e0ad5fc 100644 --- a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java +++ b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java @@ -1,15 +1,37 @@ package com.example.mtusi_messenger.repository; import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.Set; +import java.time.Duration; +import java.time.Instant; public interface AttachmentRepository { - Attachment save(Attachment attachment); + Attachment saveQueued(Attachment attachment, AttachmentJob job); Optional findById(UUID id); List findAllByIdsInChat(UUID chatId, List ids); Set findAllObjectKeys(); + Set findActiveStagingKeys(); + void markPublished(UUID jobId); + boolean markPublishFaultInjected(UUID jobId); + List findUnpublishedJobs(int limit); + int currentAttempt(UUID jobId); + boolean claim(UUID jobId, UUID attachmentId, int attempt, UUID leaseToken, Duration leaseDuration); + boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken, + int nextAttempt, String errorCode, Instant nextAttemptAt); + CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion, + List variants, boolean injectCrashAfterCommit); + boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode); + List findExpiredJobs(int limit); + boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt, + String errorCode, Instant nextAttemptAt); + boolean failExpired(UUID jobId, UUID attachmentId, String errorCode); + + record CompletionResult(boolean completed, boolean crashAfterCommit) { + } } diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java index e56e4461..93794691 100644 --- a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java @@ -1,22 +1,28 @@ package com.example.mtusi_messenger.repository; import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.UUID; import java.util.Set; -import java.util.HashSet; +import java.util.UUID; @Repository public class PostgresAttachmentRepository implements AttachmentRepository { private static final String COLUMNS = """ id, chat_id, owner_id, filename, content_type, size_bytes, - sha256, object_key, created_at, updated_at + sha256, object_key, staging_key, status, error_code, created_at, updated_at """; private final JdbcTemplate jdbc; @@ -25,16 +31,22 @@ public class PostgresAttachmentRepository implements AttachmentRepository { } @Override - public Attachment save(Attachment attachment) { + public Attachment saveQueued(Attachment attachment, AttachmentJob job) { jdbc.update(""" INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, - sha256, object_key, status, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'ready', ?, ?) + sha256, object_key, staging_key, status, error_code, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?) """, attachment.id(), attachment.chatId(), attachment.ownerId(), attachment.filename(), attachment.contentType(), attachment.sizeBytes(), - attachment.sha256(), attachment.objectKey(), Timestamp.from(attachment.createdAt()), - Timestamp.from(attachment.updatedAt())); + attachment.sha256(), attachment.objectKey(), attachment.stagingKey(), + Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt())); + jdbc.update(""" + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, attempt, status, published, created_at, updated_at) + VALUES (?, ?, ?, ?, 'queued', false, ?, ?) + """, job.jobId(), job.attachmentId(), job.pipelineVersion(), job.attempt(), + Timestamp.from(job.createdAt()), Timestamp.from(job.createdAt())); return attachment; } @@ -56,19 +68,236 @@ public class PostgresAttachmentRepository implements AttachmentRepository { args[i + 1] = ids.get(i); } return jdbc.query("SELECT " + COLUMNS + " FROM attachments WHERE chat_id = ? AND id IN (" - + placeholders + ") AND status = 'ready'", + + placeholders + ") AND status <> 'failed'", (rs, rowNum) -> map(rs), args); } @Override public Set findAllObjectKeys() { - return new HashSet<>(jdbc.query("SELECT object_key FROM attachments", + Set keys = new HashSet<>(jdbc.query( + "SELECT object_key FROM attachments WHERE status = 'ready'", (rs, rowNum) -> rs.getString("object_key"))); + keys.addAll(jdbc.query("SELECT object_key FROM attachment_variants", + (rs, rowNum) -> rs.getString("object_key"))); + return keys; } - private static Attachment map(java.sql.ResultSet rs) throws java.sql.SQLException { + @Override + public Set findActiveStagingKeys() { + return new HashSet<>(jdbc.query(""" + SELECT staging_key + FROM attachments + WHERE status IN ('queued', 'processing') AND staging_key IS NOT NULL + """, (rs, rowNum) -> rs.getString("staging_key"))); + } + + @Override + public void markPublished(UUID jobId) { + jdbc.update(""" + UPDATE attachment_jobs + SET published = true, updated_at = clock_timestamp() + WHERE id = ? AND status = 'queued' + """, jobId); + } + + @Override + public boolean markPublishFaultInjected(UUID jobId) { + return jdbc.update(""" + UPDATE attachment_jobs + SET publish_fault_injected = true, updated_at = clock_timestamp() + WHERE id = ? AND publish_fault_injected = false AND status = 'queued' + """, jobId) == 1; + } + + @Override + public List findUnpublishedJobs(int limit) { + return jdbc.query(""" + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE status = 'queued' AND published = false + AND next_attempt_at <= clock_timestamp() + ORDER BY next_attempt_at, created_at, id + LIMIT ? + """, (rs, rowNum) -> mapJob(rs), limit); + } + + @Override + public int currentAttempt(UUID jobId) { + Integer attempt = jdbc.query("SELECT attempt FROM attachment_jobs WHERE id = ?", + (rs, rowNum) -> rs.getInt("attempt"), jobId).stream().findFirst().orElse(null); + return attempt == null ? 1 : attempt; + } + + @Override + public boolean claim(UUID jobId, UUID attachmentId, int attempt, + UUID leaseToken, Duration leaseDuration) { + int claimed = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'processing', attempt = GREATEST(attempt, ?), + lease_token = ?, + lease_until = clock_timestamp() + (? * interval '1 millisecond'), + updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? + AND (status = 'queued' + OR (status = 'processing' AND lease_until < clock_timestamp())) + """, attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId); + if (claimed == 0) { + return false; + } + jdbc.update(""" + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() + WHERE id = ? AND status NOT IN ('ready', 'failed') + """, attachmentId); + return true; + } + + @Override + public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken, + int nextAttempt, String errorCode, Instant nextAttemptAt) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'queued', attempt = ?, published = false, + last_error_code = ?, next_attempt_at = ?, + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND lease_token = ? + """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), + jobId, attachmentId, leaseToken); + if (updated == 0) { + return false; + } + jdbc.update(""" + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = ? AND status NOT IN ('ready', 'failed') + """, attachmentId); + return true; + } + + @Override + public CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion, + List variants, boolean injectCrashAfterCommit) { + boolean triggerCrash = injectCrashAfterCommit && Boolean.TRUE.equals(jdbc.query(""" + SELECT NOT result_fault_injected + FROM attachment_jobs + WHERE id = ? AND attachment_id = ? AND status = 'processing' AND lease_token = ? + FOR UPDATE + """, (rs, rowNum) -> rs.getBoolean(1), jobId, attachmentId, leaseToken) + .stream().findFirst().orElse(false)); + int completed = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'ready', published = true, last_error_code = NULL, + lease_token = NULL, lease_until = NULL, + result_fault_injected = result_fault_injected OR ?, + updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND lease_token = ? + """, triggerCrash, jobId, attachmentId, leaseToken); + if (completed == 0) { + return new CompletionResult(false, false); + } + for (AttachmentVariant variant : variants) { + jdbc.update(""" + INSERT INTO attachment_variants + (attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp()) + ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + object_key = EXCLUDED.object_key + """, attachmentId, pipelineVersion, variant.name(), variant.contentType(), + variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(), + variant.objectKey()); + } + jdbc.update(""" + UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() + WHERE id = ? AND status <> 'failed' + """, attachmentId); + return new CompletionResult(true, triggerCrash); + } + + @Override + public boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'failed', published = true, last_error_code = ?, + lease_token = NULL, lease_until = NULL, + updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND lease_token = ? + """, errorCode, jobId, attachmentId, leaseToken); + if (updated == 0) { + return false; + } + jdbc.update(""" + UPDATE attachments + SET status = 'failed', error_code = ?, updated_at = clock_timestamp() + WHERE id = ? AND status <> 'ready' + """, errorCode, attachmentId); + return true; + } + + @Override + public List findExpiredJobs(int limit) { + return jdbc.query(""" + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE status = 'processing' AND lease_until < clock_timestamp() + ORDER BY lease_until, id + LIMIT ? + """, (rs, rowNum) -> mapJob(rs), limit); + } + + @Override + public boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt, + String errorCode, Instant nextAttemptAt) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'queued', attempt = ?, published = false, + last_error_code = ?, next_attempt_at = ?, + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND lease_until < clock_timestamp() + """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId); + if (updated == 1) { + jdbc.update(""" + UPDATE attachments SET status = 'queued', error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? AND status = 'processing' + """, attachmentId); + } + return updated == 1; + } + + @Override + public boolean failExpired(UUID jobId, UUID attachmentId, String errorCode) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'failed', published = true, last_error_code = ?, + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND lease_until < clock_timestamp() + """, errorCode, jobId, attachmentId); + if (updated == 1) { + jdbc.update(""" + UPDATE attachments SET status = 'failed', error_code = ?, + updated_at = clock_timestamp() + WHERE id = ? AND status <> 'ready' + """, errorCode, attachmentId); + } + return updated == 1; + } + + private Attachment map(ResultSet rs) throws SQLException { + UUID id = rs.getObject("id", UUID.class); return new Attachment( - rs.getObject("id", UUID.class), + id, rs.getObject("chat_id", UUID.class), rs.getObject("owner_id", UUID.class), rs.getString("filename"), @@ -76,7 +305,34 @@ public class PostgresAttachmentRepository implements AttachmentRepository { rs.getLong("size_bytes"), rs.getString("sha256"), rs.getString("object_key"), + rs.getString("staging_key"), + rs.getString("status"), + findVariants(id), + rs.getString("error_code"), rs.getTimestamp("created_at").toInstant(), rs.getTimestamp("updated_at").toInstant()); } + + private List findVariants(UUID attachmentId) { + return jdbc.query(""" + SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = ? AND pipeline_version = 1 + ORDER BY name + """, (rs, rowNum) -> new AttachmentVariant( + rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"), + rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"), + rs.getString("object_key")), attachmentId); + } + + private static AttachmentJob mapJob(ResultSet rs) throws SQLException { + return new AttachmentJob( + AttachmentJob.SCHEMA_VERSION, + rs.getObject("id", UUID.class), + AttachmentJob.TYPE, + rs.getObject("attachment_id", UUID.class), + rs.getInt("pipeline_version"), + rs.getInt("attempt"), + rs.getTimestamp("created_at").toInstant()); + } } diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java index 4e45794b..415abb0e 100644 --- a/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java @@ -6,6 +6,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -16,6 +17,7 @@ import java.util.HashSet; import java.util.Set; @Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) public class AttachmentReconciler { private static final Logger log = LoggerFactory.getLogger(AttachmentReconciler.class); private static final String PREFIX = "attachments/"; diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java index 83a15875..56d7bf63 100644 --- a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java @@ -2,6 +2,9 @@ package com.example.mtusi_messenger.service; import com.example.mtusi_messenger.exception.ApiExceptions; import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; +import com.example.mtusi_messenger.queue.AttachmentJobPublisher; import com.example.mtusi_messenger.repository.AttachmentRepository; import com.example.mtusi_messenger.storage.ObjectStorage; import com.example.mtusi_messenger.storage.ObjectStorageException; @@ -34,25 +37,30 @@ public class AttachmentService { private final AttachmentRepository attachments; private final ChatService chats; private final ObjectStorage storage; + private final AttachmentJobPublisher publisher; private final TransactionTemplate transaction; private final long maxUploadBytes; private final long maxImagePixels; private final Semaphore uploadPermits; private final long uploadPermitWaitMillis; + private final boolean injectCrashAfterOutboxCommit; public AttachmentService( AttachmentRepository attachments, ChatService chats, ObjectStorage storage, + AttachmentJobPublisher publisher, TransactionTemplate transaction, @Value("${messenger.attachments.max-upload-bytes:10485760}") long maxUploadBytes, @Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels, @Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads, - @Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis + @Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis, + @Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit ) { this.attachments = attachments; this.chats = chats; this.storage = storage; + this.publisher = publisher; this.transaction = transaction; this.maxUploadBytes = maxUploadBytes; this.maxImagePixels = maxImagePixels; @@ -61,6 +69,7 @@ public class AttachmentService { } this.uploadPermits = new Semaphore(maxConcurrentUploads, true); this.uploadPermitWaitMillis = uploadPermitWaitMillis; + this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit; } public Attachment upload(UUID chatId, @@ -108,16 +117,36 @@ public class AttachmentService { UUID id = UuidCreator.getTimeOrderedEpochPlus1(); String objectKey = "attachments/" + id + "/original"; + String stagingKey = "staging/" + id + "/original"; Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS); + AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, now); Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType, - bytes.length, sha256(bytes), objectKey, now, now); + bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(), + null, now, now); - storage.put(objectKey, bytes, contentType); + storage.put(stagingKey, bytes, contentType); try { - return transaction.execute(status -> attachments.save(attachment)); - } catch (RuntimeException databaseFailure) { + Attachment saved = transaction.execute(status -> attachments.saveQueued(attachment, job)); + if (injectCrashAfterOutboxCommit && attachments.markPublishFaultInjected(job.jobId())) { + // Opt-in fault injection for the exact DB COMMIT -> broker publish window. + // The persisted outbox row remains unpublished and is recovered after restart. + Runtime.getRuntime().halt(87); + } try { - storage.delete(objectKey); + publisher.publish(job); + attachments.markPublished(job.jobId()); + return saved; + } catch (RuntimeException publishFailure) { + // The database job and staging object are intentionally retained. + // The scheduled republisher closes the DB-commit -> broker window. + throw ApiExceptions.unavailable("Attachment was staged; queue publication is pending"); + } + } catch (RuntimeException databaseFailure) { + if (databaseFailure instanceof com.example.mtusi_messenger.exception.ApiException) { + throw databaseFailure; + } + try { + storage.delete(stagingKey); } catch (RuntimeException cleanupFailure) { databaseFailure.addSuppressed(cleanupFailure); } @@ -133,8 +162,28 @@ public class AttachmentService { } public Download download(UUID attachmentId, UUID userId) { + return download(attachmentId, userId, "original"); + } + + public Download download(UUID attachmentId, UUID userId, String variantName) { Attachment attachment = getAccessible(attachmentId, userId); - return new Download(attachment, storage.get(attachment.objectKey())); + if (!"ready".equals(attachment.status())) { + throw ApiExceptions.conflict("Attachment content is not ready"); + } + if ("original".equals(variantName)) { + return new Download(attachment, attachment.contentType(), attachment.filename(), + storage.get(attachment.objectKey())); + } + if (!List.of("thumbnail", "crop", "optimized").contains(variantName)) { + throw ApiExceptions.invalid("Unsupported attachment variant"); + } + AttachmentVariant variant = attachment.variants().stream() + .filter(candidate -> candidate.name().equals(variantName)) + .findFirst() + .orElseThrow(() -> ApiExceptions.notFound("Attachment variant not found")); + String extension = "image/webp".equals(variant.contentType()) ? ".webp" : ".jpg"; + return new Download(attachment, variant.contentType(), variantName + extension, + storage.get(variant.objectKey())); } public void validateForMessage(UUID chatId, List attachmentIds) { @@ -234,6 +283,6 @@ public class AttachmentService { } } - public record Download(Attachment attachment, byte[] bytes) { + public record Download(Attachment attachment, String contentType, String filename, byte[] bytes) { } } diff --git a/src/main/java/com/example/mtusi_messenger/service/StagingCleaner.java b/src/main/java/com/example/mtusi_messenger/service/StagingCleaner.java new file mode 100644 index 00000000..f347f895 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/StagingCleaner.java @@ -0,0 +1,65 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Clock; +import java.time.Duration; +import java.util.Set; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class StagingCleaner { + private static final Logger log = LoggerFactory.getLogger(StagingCleaner.class); + private static final String PREFIX = "staging/"; + private final AttachmentRepository attachments; + private final ObjectStorage storage; + private final Duration retention; + private final Clock clock; + + @Autowired + public StagingCleaner( + AttachmentRepository attachments, + ObjectStorage storage, + @Value("${messenger.attachments.staging-retention-seconds:86400}") long retentionSeconds) { + this(attachments, storage, Duration.ofSeconds(retentionSeconds), Clock.systemUTC()); + } + + StagingCleaner( + AttachmentRepository attachments, + ObjectStorage storage, + Duration retention, + Clock clock) { + if (retention.isZero() || retention.isNegative()) { + throw new IllegalArgumentException("Staging retention must be positive"); + } + this.attachments = attachments; + this.storage = storage; + this.retention = retention; + this.clock = clock; + } + + @Scheduled( + fixedDelayString = "${messenger.attachments.reconcile-interval-millis:600000}", + initialDelayString = "${messenger.attachments.reconcile-initial-delay-millis:60000}") + public void cleanup() { + try { + Set active = attachments.findActiveStagingKeys(); + var deleteBefore = clock.instant().minus(retention); + for (ObjectStorage.StoredObject object : storage.list(PREFIX)) { + if (!active.contains(object.objectKey()) && object.lastModified().isBefore(deleteBefore)) { + storage.delete(object.objectKey()); + } + } + } catch (RuntimeException error) { + log.warn("Staging cleanup postponed: {}", error.getMessage()); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java b/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java new file mode 100644 index 00000000..571e475f --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java @@ -0,0 +1,239 @@ +package com.example.mtusi_messenger.worker; + +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; +import com.example.mtusi_messenger.queue.RetryPolicy; +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.rabbitmq.client.Channel; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "worker") +public class AttachmentWorker { + private static final Logger log = LoggerFactory.getLogger(AttachmentWorker.class); + private final ObjectMapper mapper; + private final AttachmentRepository attachments; + private final ObjectStorage storage; + private final ImageVariantProcessor images; + private final RetryPolicy retryPolicy; + private final TransactionTemplate transaction; + private final int maxAttempts; + private final Duration jobTimeout; + private final Duration leaseDuration; + private final boolean injectCrashAfterResultCommit; + private final ExecutorService processingPool; + private final Clock clock = Clock.systemUTC(); + + public AttachmentWorker( + ObjectMapper mapper, + AttachmentRepository attachments, + ObjectStorage storage, + ImageVariantProcessor images, + RetryPolicy retryPolicy, + TransactionTemplate transaction, + @Value("${messenger.jobs.max-attempts:5}") int maxAttempts, + @Value("${messenger.jobs.timeout:60s}") Duration jobTimeout, + @Value("${messenger.jobs.lease-duration:90s}") Duration leaseDuration, + @Value("${messenger.jobs.fault-crash-after-result-commit:false}") boolean injectCrashAfterResultCommit, + @Value("${spring.rabbitmq.listener.simple.concurrency:2}") int concurrency) { + if (maxAttempts < 1 || jobTimeout.isZero() || jobTimeout.isNegative() + || leaseDuration.compareTo(jobTimeout) <= 0 || concurrency < 1) { + throw new IllegalArgumentException("Invalid attachment worker limits"); + } + this.mapper = mapper; + this.attachments = attachments; + this.storage = storage; + this.images = images; + this.retryPolicy = retryPolicy; + this.transaction = transaction; + this.maxAttempts = maxAttempts; + this.jobTimeout = jobTimeout; + this.leaseDuration = leaseDuration; + this.injectCrashAfterResultCommit = injectCrashAfterResultCommit; + this.processingPool = Executors.newFixedThreadPool(concurrency, + Thread.ofPlatform().name("attachment-image-", 0).factory()); + } + + @RabbitListener(queues = "${messenger.jobs.queue}") + public void consume(Message message, Channel channel) throws IOException { + long deliveryTag = message.getMessageProperties().getDeliveryTag(); + AttachmentJob job; + try { + job = mapper.readValue(message.getBody(), AttachmentJob.class); + } catch (RuntimeException | IOException invalidMessage) { + log.error("Rejecting malformed attachment job message {}", + message.getMessageProperties().getMessageId()); + channel.basicReject(deliveryTag, false); + return; + } + + Attachment attachment = attachments.findById(job.attachmentId()).orElse(null); + if (attachment == null || "ready".equals(attachment.status()) || "failed".equals(attachment.status())) { + channel.basicAck(deliveryTag, false); + return; + } + + int attempt = Math.max(job.attempt(), attachments.currentAttempt(job.jobId())); + UUID leaseToken = UUID.randomUUID(); + boolean claimed = Boolean.TRUE.equals(transaction.execute(status -> + attachments.claim(job.jobId(), job.attachmentId(), attempt, leaseToken, leaseDuration))); + if (!claimed) { + // A live worker owns the lease, or a terminal state won the race. + // If the owner died, the DB lease reaper will create a new outbox delivery. + channel.basicAck(deliveryTag, false); + return; + } + + Future future = processingPool.submit(() -> process(job, leaseToken)); + try { + ProcessOutcome outcome = future.get(jobTimeout.toMillis(), TimeUnit.MILLISECONDS); + if (outcome.crashAfterCommit()) { + log.error("Fault injection: halting worker after durable result and before consumer ack"); + Runtime.getRuntime().halt(86); + } + channel.basicAck(deliveryTag, false); + } catch (TimeoutException timeout) { + future.cancel(true); + retryOrFail(job, leaseToken, attempt, "processing_timeout", deliveryTag, channel); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + future.cancel(true); + channel.basicNack(deliveryTag, false, true); + } catch (ExecutionException execution) { + Throwable cause = execution.getCause(); + if (cause instanceof ImageVariantProcessor.ProcessingException imageFailure + && imageFailure.permanent()) { + failPermanent(job, leaseToken, imageFailure.errorCode(), deliveryTag, channel); + } else { + String code = cause instanceof ImageVariantProcessor.ProcessingException imageFailure + ? imageFailure.errorCode() : "processing_failed"; + retryOrFail(job, leaseToken, attempt, code, deliveryTag, channel); + } + } + } + + private ProcessOutcome process(AttachmentJob job, UUID leaseToken) { + Attachment attachment = attachments.findById(job.attachmentId()) + .orElseThrow(() -> new ImageVariantProcessor.ProcessingException( + "attachment_missing", true, "Attachment metadata is missing", null)); + byte[] original = storage.get(attachment.stagingKey()); + if (original.length != attachment.sizeBytes() || !sha256(original).equals(attachment.sha256())) { + throw new ImageVariantProcessor.ProcessingException( + "invalid_image", true, "Staging checksum mismatch", null); + } + + storage.put(attachment.objectKey(), original, attachment.contentType()); + List generated = images.createVariants(original); + List variants = new ArrayList<>(generated.size()); + for (ImageVariantProcessor.GeneratedVariant item : generated) { + String objectKey = "attachments/" + attachment.id() + "/" + + item.name() + "-v" + job.pipelineVersion() + "." + item.extension(); + storage.put(objectKey, item.bytes(), item.contentType()); + variants.add(new AttachmentVariant( + item.name(), item.contentType(), item.bytes().length, + item.width(), item.height(), sha256(item.bytes()), objectKey)); + } + + AttachmentRepository.CompletionResult completion = transaction.execute(status -> + attachments.markReady(job.jobId(), job.attachmentId(), leaseToken, + job.pipelineVersion(), variants, injectCrashAfterResultCommit)); + if (completion == null || !completion.completed()) { + return new ProcessOutcome(false); + } + try { + storage.delete(attachment.stagingKey()); + } catch (RuntimeException cleanupFailure) { + log.warn("Staging cleanup postponed for attachment {}", attachment.id()); + } + return new ProcessOutcome(completion.crashAfterCommit()); + } + + private void failPermanent( + AttachmentJob job, UUID leaseToken, String errorCode, + long deliveryTag, Channel channel) throws IOException { + boolean failed = Boolean.TRUE.equals(transaction.execute(status -> + attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, errorCode))); + if (failed) { + channel.basicReject(deliveryTag, false); + } else { + channel.basicAck(deliveryTag, false); + } + } + + private void retryOrFail( + AttachmentJob job, UUID leaseToken, int attempt, String errorCode, + long deliveryTag, Channel channel) throws IOException { + try { + if (attempt >= maxAttempts) { + boolean failed = Boolean.TRUE.equals(transaction.execute(status -> + attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, + "retry_exhausted"))); + if (failed) { + log.error("Attachment job {} exhausted {} attempts: {}", + job.jobId(), attempt, errorCode); + channel.basicReject(deliveryTag, false); + } else { + channel.basicAck(deliveryTag, false); + } + return; + } + + int nextAttempt = attempt + 1; + boolean queued = Boolean.TRUE.equals(transaction.execute(status -> + attachments.markQueuedForRetry( + job.jobId(), job.attachmentId(), leaseToken, nextAttempt, errorCode, + clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt))))); + // The DB outbox is now the durable retry carrier; the current broker + // delivery can be acknowledged without a DB/broker dual write. + channel.basicAck(deliveryTag, false); + if (!queued) { + log.info("Retry skipped because job {} lost its lease or became terminal", job.jobId()); + } + } catch (RuntimeException databaseFailure) { + channel.basicNack(deliveryTag, false, true); + } + } + + @PreDestroy + void shutdownPool() { + processingPool.shutdownNow(); + } + + private static String sha256(byte[] bytes) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + private record ProcessOutcome(boolean crashAfterCommit) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java b/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java new file mode 100644 index 00000000..cb077cae --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java @@ -0,0 +1,254 @@ +package com.example.mtusi_messenger.worker; + +import net.coobird.thumbnailator.Thumbnails; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.imageio.IIOImage; +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.ImageWriteParam; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +@Component +public class ImageVariantProcessor { + private static final long MEBIBYTE = 1024L * 1024L; + private static final long OUTPUT_ALLOWANCE_BYTES = 64L * MEBIBYTE; + private final long maxPixels; + private final long memoryBudgetBytes; + private final Semaphore memoryPermits; + private final Duration memoryWait; + + public ImageVariantProcessor( + @Value("${messenger.jobs.worker-max-pixels:40000000}") long maxPixels, + @Value("${messenger.jobs.worker-memory-budget-bytes:536870912}") long memoryBudgetBytes, + @Value("${messenger.jobs.memory-wait:2s}") Duration memoryWait) { + if (maxPixels < 1 || memoryBudgetBytes < MEBIBYTE || memoryWait.isNegative()) { + throw new IllegalArgumentException("Invalid worker image resource limits"); + } + this.maxPixels = maxPixels; + this.memoryBudgetBytes = memoryBudgetBytes; + this.memoryPermits = new Semaphore((int) Math.min(Integer.MAX_VALUE, + Math.max(1, memoryBudgetBytes / MEBIBYTE)), true); + this.memoryWait = memoryWait; + } + + public List createVariants(byte[] original) { + ImageInfo info = inspect(original); + long estimatedBytes; + try { + estimatedBytes = Math.addExact(Math.multiplyExact(info.pixels(), 12L), + OUTPUT_ALLOWANCE_BYTES); + } catch (ArithmeticException overflow) { + throw permanent("image_too_large", "Image memory estimate overflow", overflow); + } + if (estimatedBytes > memoryBudgetBytes) { + throw permanent("image_too_large", "Image exceeds worker memory budget", null); + } + int permits = (int) Math.max(1, (estimatedBytes + MEBIBYTE - 1) / MEBIBYTE); + boolean acquired; + try { + acquired = memoryPermits.tryAcquire(permits, memoryWait.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw temporary("worker_interrupted", "Interrupted while waiting for memory budget", interrupted); + } + if (!acquired) { + throw temporary("worker_capacity", "Worker memory budget is currently exhausted", null); + } + + try { + return transform(original); + } finally { + memoryPermits.release(permits); + } + } + + private ImageInfo inspect(byte[] bytes) { + ImageIO.setUseCache(false); + try (var input = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) { + if (input == null) { + throw permanent("invalid_image", "Image input is invalid", null); + } + Iterator readers = ImageIO.getImageReaders(input); + if (!readers.hasNext()) { + throw permanent("invalid_image", "Unsupported image encoding", null); + } + ImageReader reader = readers.next(); + try { + reader.setInput(input, true, true); + int width = reader.getWidth(0); + int height = reader.getHeight(0); + long pixels = Math.multiplyExact((long) width, height); + if (width < 1 || height < 1 || pixels > maxPixels) { + throw permanent("image_too_large", "Image exceeds worker pixel limit", null); + } + return new ImageInfo(width, height, pixels); + } finally { + reader.dispose(); + } + } catch (ProcessingException error) { + throw error; + } catch (IOException | RuntimeException error) { + throw permanent("invalid_image", "Image header cannot be read", error); + } + } + + private List transform(byte[] original) { + try { + BufferedImage oriented = Thumbnails.of(new ByteArrayInputStream(original)) + .useExifOrientation(true) + .scale(1.0) + .asBufferedImage(); + if (oriented == null || oriented.getWidth() < 1 || oriented.getHeight() < 1) { + throw permanent("invalid_image", "Image cannot be decoded", null); + } + try { + List variants = new ArrayList<>(3); + BufferedImage thumbnail = fit(oriented, 256, BufferedImage.TYPE_INT_RGB, true); + variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", 0.85f)); + thumbnail.flush(); + + BufferedImage crop = cropSquare(oriented, 128); + variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", 0.88f)); + crop.flush(); + + int webpType = oriented.getColorModel().hasAlpha() + ? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB; + BufferedImage optimized = fit(oriented, 1280, webpType, !oriented.getColorModel().hasAlpha()); + variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", 0.82f)); + optimized.flush(); + return List.copyOf(variants); + } finally { + oriented.flush(); + } + } catch (ProcessingException error) { + throw error; + } catch (IOException | RuntimeException error) { + throw permanent("invalid_image", "Image cannot be transformed", error); + } + } + + private static BufferedImage fit(BufferedImage source, int maxSide, int type, boolean whiteBackground) { + double scale = Math.min(1.0, Math.min( + (double) maxSide / source.getWidth(), + (double) maxSide / source.getHeight())); + int width = Math.max(1, (int) Math.floor(source.getWidth() * scale)); + int height = Math.max(1, (int) Math.floor(source.getHeight() * scale)); + BufferedImage target = new BufferedImage(width, height, type); + Graphics2D graphics = qualityGraphics(target, whiteBackground); + try { + graphics.drawImage(source, 0, 0, width, height, null); + } finally { + graphics.dispose(); + } + return target; + } + + private static BufferedImage cropSquare(BufferedImage source, int side) { + int sourceSide = Math.min(source.getWidth(), source.getHeight()); + int x = (source.getWidth() - sourceSide) / 2; + int y = (source.getHeight() - sourceSide) / 2; + BufferedImage target = new BufferedImage(side, side, BufferedImage.TYPE_INT_RGB); + Graphics2D graphics = qualityGraphics(target, true); + try { + graphics.drawImage(source, 0, 0, side, side, + x, y, x + sourceSide, y + sourceSide, null); + } finally { + graphics.dispose(); + } + return target; + } + + private static Graphics2D qualityGraphics(BufferedImage target, boolean whiteBackground) { + Graphics2D graphics = target.createGraphics(); + if (whiteBackground) { + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, target.getWidth(), target.getHeight()); + } + graphics.setRenderingHint(RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BICUBIC); + graphics.setRenderingHint(RenderingHints.KEY_RENDERING, + RenderingHints.VALUE_RENDER_QUALITY); + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); + return graphics; + } + + private static GeneratedVariant encode( + String name, String contentType, String extension, + BufferedImage image, String format, float quality) throws IOException { + Iterator writers = ImageIO.getImageWritersByFormatName(format); + if (!writers.hasNext()) { + throw new IOException(format + " writer is unavailable"); + } + var writer = writers.next(); + try (var output = new ByteArrayOutputStream(); + var imageOutput = ImageIO.createImageOutputStream(output)) { + writer.setOutput(imageOutput); + ImageWriteParam parameters = writer.getDefaultWriteParam(); + if (parameters.canWriteCompressed()) { + parameters.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); + String[] compressionTypes = parameters.getCompressionTypes(); + if (compressionTypes != null && compressionTypes.length > 0) { + parameters.setCompressionType(compressionTypes[0]); + } + parameters.setCompressionQuality(quality); + } + // Source metadata is intentionally omitted for every generated variant. + writer.write(null, new IIOImage(image, null, null), parameters); + imageOutput.flush(); + return new GeneratedVariant(name, contentType, extension, output.toByteArray(), + image.getWidth(), image.getHeight()); + } finally { + writer.dispose(); + } + } + + private static ProcessingException permanent(String code, String message, Throwable cause) { + return new ProcessingException(code, true, message, cause); + } + + private static ProcessingException temporary(String code, String message, Throwable cause) { + return new ProcessingException(code, false, message, cause); + } + + private record ImageInfo(int width, int height, long pixels) { + } + + public record GeneratedVariant(String name, String contentType, String extension, + byte[] bytes, int width, int height) { + } + + public static final class ProcessingException extends RuntimeException { + private final String errorCode; + private final boolean permanent; + + public ProcessingException( + String errorCode, boolean permanent, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + this.permanent = permanent; + } + + public String errorCode() { + return errorCode; + } + + public boolean permanent() { + return permanent; + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6a55c26a..10f92def 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -27,6 +27,23 @@ spring: min-idle: ${REDIS_POOL_MIN_IDLE:0} max-wait: ${REDIS_POOL_MAX_WAIT:500ms} + rabbitmq: + host: ${RABBITMQ_HOST:localhost} + port: ${RABBITMQ_PORT:5672} + username: ${RABBITMQ_USER:messenger} + password: ${RABBITMQ_PASSWORD:messenger} + virtual-host: ${RABBITMQ_VHOST:/} + connection-timeout: ${RABBITMQ_CONNECTION_TIMEOUT:5s} + publisher-confirm-type: correlated + publisher-returns: true + listener: + simple: + acknowledge-mode: manual + default-requeue-rejected: false + prefetch: ${WORKER_PREFETCH:1} + concurrency: ${WORKER_CONCURRENCY:2} + max-concurrency: ${WORKER_CONCURRENCY:2} + jdbc: template: query-timeout: ${DATABASE_QUERY_TIMEOUT_SECONDS:5} @@ -53,6 +70,7 @@ server: shutdown: graceful messenger: + role: ${MESSENGER_ROLE:api} storage: postgres session: ttl-seconds: ${SESSION_TTL_SECONDS:3600} @@ -71,7 +89,28 @@ messenger: max-image-pixels: ${MAX_IMAGE_PIXELS:40000000} max-concurrent-uploads: ${MAX_CONCURRENT_UPLOADS:3} upload-permit-wait-millis: ${UPLOAD_PERMIT_WAIT_MILLIS:10000} + staging-retention-seconds: ${STAGING_RETENTION_SECONDS:86400} cleanup-grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600} cleanup-dry-run: ${ATTACHMENT_CLEANUP_DRY_RUN:false} reconcile-interval-millis: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:600000} reconcile-initial-delay-millis: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:60000} + jobs: + exchange: ${ATTACHMENT_JOB_EXCHANGE:attachments.v1} + queue: ${ATTACHMENT_JOB_QUEUE:attachments.process.level4.v1} + routing-key: ${ATTACHMENT_JOB_ROUTING_KEY:attachment.process.v1} + dead-letter-exchange: ${ATTACHMENT_DLQ_EXCHANGE:attachments.dlx.v1} + dead-letter-queue: ${ATTACHMENT_DLQ_QUEUE:attachments.failed.v1} + dead-letter-routing-key: ${ATTACHMENT_DLQ_ROUTING_KEY:attachment.failed.v1} + max-attempts: ${JOB_MAX_ATTEMPTS:5} + retry-base-delay: ${JOB_RETRY_BASE_DELAY:1s} + retry-max-delay: ${JOB_RETRY_MAX_DELAY:30s} + timeout: ${JOB_TIMEOUT:60s} + lease-duration: ${JOB_LEASE_DURATION:90s} + lease-reaper-interval: ${JOB_LEASE_REAPER_INTERVAL:5s} + worker-max-pixels: ${WORKER_MAX_IMAGE_PIXELS:40000000} + worker-memory-budget-bytes: ${WORKER_MEMORY_BUDGET_BYTES:536870912} + memory-wait: ${WORKER_MEMORY_WAIT:2s} + publish-confirm-timeout: ${JOB_PUBLISH_CONFIRM_TIMEOUT:5s} + republish-interval: ${JOB_REPUBLISH_INTERVAL:5s} + fault-crash-after-outbox-commit: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:false} + fault-crash-after-result-commit: ${FAULT_CRASH_AFTER_RESULT_COMMIT:false} diff --git a/src/main/resources/db/migration/V5__add_attachment_processing_jobs.sql b/src/main/resources/db/migration/V5__add_attachment_processing_jobs.sql new file mode 100644 index 00000000..5d5ecc40 --- /dev/null +++ b/src/main/resources/db/migration/V5__add_attachment_processing_jobs.sql @@ -0,0 +1,58 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk; +ALTER TABLE attachments + ADD COLUMN staging_key text, + ADD COLUMN error_code varchar(64), + ADD CONSTRAINT attachments_status_chk + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD CONSTRAINT attachments_error_state_chk + CHECK ((status = 'failed' AND error_code IS NOT NULL) + OR (status <> 'failed' AND error_code IS NULL)); + +CREATE UNIQUE INDEX attachments_staging_key_uk + ON attachments (staging_key) WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name varchar(32) NOT NULL, + content_type varchar(32) NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 char(64) NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL, + + CONSTRAINT attachment_variants_pk + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_uk UNIQUE (object_key), + CONSTRAINT attachment_variants_name_chk + CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_content_type_chk + CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + CONSTRAINT attachment_variants_size_chk CHECK (size_bytes > 0), + CONSTRAINT attachment_variants_dimensions_chk CHECK (width > 0 AND height > 0), + CONSTRAINT attachment_variants_sha256_chk CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 1, + status varchar(16) NOT NULL DEFAULT 'queued', + published boolean NOT NULL DEFAULT false, + last_error_code varchar(64), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + + CONSTRAINT attachment_jobs_pipeline_chk CHECK (pipeline_version > 0), + CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt > 0), + CONSTRAINT attachment_jobs_status_chk + CHECK (status IN ('queued', 'processing', 'ready', 'failed')) +); + +CREATE INDEX attachment_jobs_publish_idx + ON attachment_jobs (published, status, created_at) + WHERE published = false AND status = 'queued'; + diff --git a/src/main/resources/db/migration/V6__add_job_leases_replay_and_fault_injection.sql b/src/main/resources/db/migration/V6__add_job_leases_replay_and_fault_injection.sql new file mode 100644 index 00000000..1a8539c7 --- /dev/null +++ b/src/main/resources/db/migration/V6__add_job_leases_replay_and_fault_injection.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachment_jobs + ADD COLUMN next_attempt_at timestamptz, + ADD COLUMN lease_token uuid, + ADD COLUMN lease_until timestamptz, + ADD COLUMN publish_fault_injected boolean NOT NULL DEFAULT false, + ADD COLUMN result_fault_injected boolean NOT NULL DEFAULT false; + +UPDATE attachment_jobs SET next_attempt_at = created_at WHERE next_attempt_at IS NULL; +UPDATE attachment_jobs +SET status = 'queued', published = false, last_error_code = 'lease_migration_recovery' +WHERE status = 'processing'; +UPDATE attachments SET status = 'queued', error_code = NULL WHERE status = 'processing'; +ALTER TABLE attachment_jobs ALTER COLUMN next_attempt_at SET NOT NULL; +ALTER TABLE attachment_jobs ALTER COLUMN next_attempt_at SET DEFAULT clock_timestamp(); + +ALTER TABLE attachment_jobs + ADD CONSTRAINT attachment_jobs_lease_chk CHECK ( + (status = 'processing' AND lease_token IS NOT NULL AND lease_until IS NOT NULL) + OR (status <> 'processing' AND lease_token IS NULL AND lease_until IS NULL) + ); + +DROP INDEX attachment_jobs_publish_idx; +CREATE INDEX attachment_jobs_publish_idx + ON attachment_jobs (next_attempt_at, created_at, id) + WHERE published = false AND status = 'queued'; +CREATE INDEX attachment_jobs_expired_lease_idx + ON attachment_jobs (lease_until) + WHERE status = 'processing'; + +CREATE TABLE attachment_job_replays ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE RESTRICT, + requested_by varchar(128) NOT NULL, + previous_attempt integer NOT NULL, + requested_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_job_replays_actor_chk + CHECK (char_length(requested_by) BETWEEN 1 AND 128) +); + +CREATE INDEX attachment_job_replays_job_idx + ON attachment_job_replays (job_id, requested_at); diff --git a/src/test/java/com/example/mtusi_messenger/queue/RetryPolicyTest.java b/src/test/java/com/example/mtusi_messenger/queue/RetryPolicyTest.java new file mode 100644 index 00000000..5ca590d8 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/queue/RetryPolicyTest.java @@ -0,0 +1,21 @@ +package com.example.mtusi_messenger.queue; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RetryPolicyTest { + @Test + void appliesExponentialBackoffWithBoundedJitter() { + RetryPolicy policy = new RetryPolicy(Duration.ofSeconds(1), Duration.ofSeconds(30)); + + for (int attempt = 2; attempt <= 8; attempt++) { + long ceiling = Math.min(30_000L, 1_000L << (attempt - 2)); + long delay = policy.delayForAttempt(attempt).toMillis(); + assertTrue(delay >= Math.max(1, ceiling / 2)); + assertTrue(delay <= ceiling); + } + } +} diff --git a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java index 8a53cb45..eb84fee2 100644 --- a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java +++ b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java @@ -4,6 +4,7 @@ import com.example.mtusi_messenger.exception.ApiException; import com.example.mtusi_messenger.model.Attachment; import com.example.mtusi_messenger.model.Chat; import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.queue.AttachmentJobPublisher; import com.example.mtusi_messenger.storage.ObjectStorage; import com.example.mtusi_messenger.storage.ObjectStorageException; import org.junit.jupiter.api.BeforeEach; @@ -34,6 +35,7 @@ class AttachmentServiceTest { private AttachmentRepository repository; private ChatService chats; private InMemoryStorage storage; + private AttachmentJobPublisher publisher; private AttachmentService service; private byte[] png; @@ -42,6 +44,7 @@ class AttachmentServiceTest { repository = mock(AttachmentRepository.class); chats = mock(ChatService.class); storage = new InMemoryStorage(); + publisher = mock(AttachmentJobPublisher.class); TransactionTemplate transaction = mock(TransactionTemplate.class); when(transaction.execute(any())).thenAnswer(invocation -> { @SuppressWarnings("unchecked") @@ -50,14 +53,14 @@ class AttachmentServiceTest { }); when(chats.getAccessible(chatId, ownerId)).thenReturn( new Chat(chatId, "test", List.of(ownerId), Instant.now())); - when(repository.save(any())).thenAnswer(invocation -> invocation.getArgument(0)); - service = new AttachmentService(repository, chats, storage, transaction, - 10_485_760, 40_000_000, 3, 10_000); + when(repository.saveQueued(any(), any())).thenAnswer(invocation -> invocation.getArgument(0)); + service = new AttachmentService(repository, chats, storage, publisher, transaction, + 10_485_760, 40_000_000, 3, 10_000, false); png = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png")); } @Test - void uploadsValidPngUsingServerGeneratedObjectKey() { + void stagesValidPngAndPublishesDurableJob() { Attachment attachment = service.upload(chatId, ownerId, "photo.png", "image/png", png.length, new ByteArrayInputStream(png)); @@ -65,9 +68,13 @@ class AttachmentServiceTest { assertEquals("image/png", attachment.contentType()); assertEquals(png.length, attachment.sizeBytes()); assertTrue(attachment.objectKey().matches("attachments/[0-9a-f-]+/original")); + assertTrue(attachment.stagingKey().matches("staging/[0-9a-f-]+/original")); + assertEquals("queued", attachment.status()); assertFalse(attachment.objectKey().contains("photo.png")); - assertArrayEquals(png, storage.objects.get(attachment.objectKey())); - verify(repository).save(attachment); + assertArrayEquals(png, storage.objects.get(attachment.stagingKey())); + assertFalse(storage.objects.containsKey(attachment.objectKey())); + verify(repository).saveQueued(eq(attachment), any()); + verify(publisher).publish(any()); } @Test @@ -108,7 +115,7 @@ class AttachmentServiceTest { assertThrows(ObjectStorageException.class, () -> service.upload( chatId, ownerId, "photo.png", "image/png", png.length, new ByteArrayInputStream(png))); - verify(repository, never()).save(any()); + verify(repository, never()).saveQueued(any(), any()); } @Test @@ -130,12 +137,12 @@ class AttachmentServiceTest { assertEquals(HttpStatus.BAD_REQUEST, error.status()); assertTrue(storage.objects.isEmpty()); - verify(repository, never()).save(any()); + verify(repository, never()).saveQueued(any(), any()); } @Test void databaseFailureCompensatesAlreadyWrittenS3Object() { - when(repository.save(any())).thenThrow(new IllegalStateException("database commit failed")); + when(repository.saveQueued(any(), any())).thenThrow(new IllegalStateException("database commit failed")); assertThrows(IllegalStateException.class, () -> service.upload( chatId, ownerId, "photo.png", "image/png", png.length, diff --git a/src/test/java/com/example/mtusi_messenger/service/StagingCleanerTest.java b/src/test/java/com/example/mtusi_messenger/service/StagingCleanerTest.java new file mode 100644 index 00000000..03c90ef1 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/service/StagingCleanerTest.java @@ -0,0 +1,55 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.junit.jupiter.api.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StagingCleanerTest { + private static final Instant NOW = Instant.parse("2026-11-18T12:00:00Z"); + + @Test + void keepsActiveAndFreshStagingButDeletesExpiredUnreferencedObject() { + AttachmentRepository repository = mock(AttachmentRepository.class); + when(repository.findActiveStagingKeys()).thenReturn(Set.of("staging/active/original")); + FakeStorage storage = new FakeStorage(List.of( + object("staging/active/original", NOW.minusSeconds(172800)), + object("staging/failed/original", NOW.minusSeconds(172800)), + object("staging/fresh/original", NOW.minusSeconds(60)))); + StagingCleaner cleaner = new StagingCleaner(repository, storage, Duration.ofDays(1), + Clock.fixed(NOW, ZoneOffset.UTC)); + + cleaner.cleanup(); + + assertEquals(List.of("staging/failed/original"), storage.deleted); + } + + private static ObjectStorage.StoredObject object(String key, Instant modified) { + return new ObjectStorage.StoredObject(key, modified, 100); + } + + private static final class FakeStorage implements ObjectStorage { + private final List objects; + private final List deleted = new ArrayList<>(); + + private FakeStorage(List objects) { + this.objects = objects; + } + + @Override public void put(String objectKey, byte[] bytes, String contentType) { } + @Override public byte[] get(String objectKey) { return new byte[0]; } + @Override public void delete(String objectKey) { deleted.add(objectKey); } + @Override public List list(String prefix) { return objects; } + } +} diff --git a/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java b/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java new file mode 100644 index 00000000..11c56b5a --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java @@ -0,0 +1,70 @@ +package com.example.mtusi_messenger.worker; + +import org.junit.jupiter.api.Test; + +import javax.imageio.ImageIO; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ImageVariantProcessorTest { + private final byte[] original; + + ImageVariantProcessorTest() throws Exception { + original = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png")); + } + + @Test + void createsThumbnailCropAndOptimizedWebp() throws Exception { + ImageVariantProcessor processor = new ImageVariantProcessor( + 40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1)); + + Map variants = processor + .createVariants(original).stream() + .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, + Function.identity())); + + assertEquals(3, variants.size()); + assertImage(variants.get("thumbnail"), "image/jpeg", 256, 160); + assertImage(variants.get("crop"), "image/jpeg", 128, 128); + assertImage(variants.get("optimized"), "image/webp", 320, 200); + byte[] webp = variants.get("optimized").bytes(); + assertEquals("RIFF", new String(webp, 0, 4, StandardCharsets.US_ASCII)); + assertEquals("WEBP", new String(webp, 8, 4, StandardCharsets.US_ASCII)); + } + + @Test + void rejectsImageWhoseEstimateExceedsWorkerMemoryBudget() { + ImageVariantProcessor processor = new ImageVariantProcessor( + 40_000_000, 1L * 1024 * 1024, Duration.ofSeconds(1)); + + var error = assertThrows(ImageVariantProcessor.ProcessingException.class, + () -> processor.createVariants(original)); + + assertTrue(error.permanent()); + assertEquals("image_too_large", error.errorCode()); + } + + private static void assertImage( + ImageVariantProcessor.GeneratedVariant variant, + String contentType, int width, int height) throws Exception { + assertNotNull(variant); + assertEquals(contentType, variant.contentType()); + assertEquals(width, variant.width()); + assertEquals(height, variant.height()); + var decoded = ImageIO.read(new ByteArrayInputStream(variant.bytes())); + assertNotNull(decoded); + assertEquals(width, decoded.getWidth()); + assertEquals(height, decoded.getHeight()); + } +} diff --git a/student_tests/README.md b/student_tests/README.md index c08c4d23..7b14c1ae 100644 --- a/student_tests/README.md +++ b/student_tests/README.md @@ -16,3 +16,9 @@ The course-provided black-box suite remains in `tests/`; Spring tests remain in unknown attachment, foreign user and cross-chat binding; every user error is 4xx. - `concurrent_upload_memory.py` — 12 simultaneous ~6.5 MiB valid PNG uploads and aggregate API container memory sampling through `docker stats`. +- `lab5_worker_recovery.py` — durable worker stop/restart, all three variants and + idempotent duplicate delivery. +- `lab5_fault_injection.py` — crashes after outbox commit and after result commit + but before consumer acknowledgement. +- `lab5_dlq_replay_lease.py` — permanent failure DLQ routing, guarded audited + replay, and expired processing-lease recovery. diff --git a/student_tests/lab5_dlq_replay_lease.py b/student_tests/lab5_dlq_replay_lease.py new file mode 100644 index 00000000..b36b9692 --- /dev/null +++ b/student_tests/lab5_dlq_replay_lease.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Exercise DLQ routing, guarded replay and expired processing leases.""" + +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] + + +def command(*args, capture=False): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, check=True, text=True, + capture_output=capture) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def env_value(name): + for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise Failure("missing " + name + " in .env.example") + + +def queue_ready(queue): + listing = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", capture=True) + for line in listing.splitlines(): + columns = line.split("\t") + if len(columns) == 2 and columns[0] == queue: + return int(columns[1]) + raise Failure("queue not found: " + queue) + + +def wait_ready(client, actor, attachment, seconds=120): + return client.wait_ready(actor, attachment, wait_seconds=seconds) + + +def replay(job_id): + subprocess.run( + ["powershell", "-NoProfile", "-File", str(ROOT / "scripts" / "replay-failed.ps1"), + "-JobId", job_id, "-RequestedBy", "integration-test", + "-EnvFile", ".env.example"], + cwd=ROOT, check=True, text=True) + + +def main(): + client = Client("http://localhost:8080") + alice, bob, _, chat = client.bootstrap() + + # A permanently malformed delivery is rejected and dead-lettered. + before_dlq = queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) + command( + "exec", "-T", "rabbitmq", "rabbitmqadmin", + "-u", env_value("RABBITMQ_USER"), "-p", env_value("RABBITMQ_PASSWORD"), + "publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"), + "routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"), + "payload={}", + 'properties={"delivery_mode":2,"content_type":"application/json"}') + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) <= before_dlq: + time.sleep(0.5) + check(queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) == before_dlq + 1, + "malformed permanent failure was not dead-lettered exactly once") + + # Replay is allowed only from failed and creates one audit record. + command("stop", "worker") + failed, _ = client.attachment(alice, chat) + job_id = database_scalar( + "SELECT id FROM attachment_jobs WHERE attachment_id = '" + failed["id"] + "';") + database_scalar( + "UPDATE attachment_jobs SET status='failed', published=true, " + "last_error_code='retry_exhausted', lease_token=NULL, lease_until=NULL " + "WHERE id='" + job_id + "'; " + "UPDATE attachments SET status='failed', error_code='retry_exhausted' " + "WHERE id='" + failed["id"] + "'; SELECT 1;") + replay(job_id) + replay(job_id) + audit_count = database_scalar( + "SELECT count(*) FROM attachment_job_replays WHERE job_id='" + job_id + "';") + check(audit_count == "1", "replay was not guarded or was audited more than once") + command("start", "worker") + ready = wait_ready(client, bob, failed) + check(ready["status"] == "ready", "replayed failed job did not become ready") + replay(job_id) + check(database_scalar( + "SELECT count(*) FROM attachment_job_replays WHERE job_id='" + job_id + "';") == "1", + "replay changed an already successful job") + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + failed["id"] + "';") == "3", + "replay did not preserve exactly one variant set") + + # The API reaper returns an expired processing lease to the durable outbox. + command("stop", "worker") + leased, _ = client.attachment(alice, chat) + lease_job = database_scalar( + "SELECT id FROM attachment_jobs WHERE attachment_id='" + leased["id"] + "';") + database_scalar( + "UPDATE attachment_jobs SET status='processing', attempt=1, published=true, " + "lease_token=gen_random_uuid(), lease_until=clock_timestamp()-interval '1 second' " + "WHERE id='" + lease_job + "'; " + "UPDATE attachments SET status='processing' WHERE id='" + leased["id"] + "'; SELECT 1;") + command("start", "worker") + lease_ready = wait_ready(client, bob, leased) + check(lease_ready["status"] == "ready", "expired processing lease was not recovered") + check(int(database_scalar( + "SELECT attempt FROM attachment_jobs WHERE id='" + lease_job + "';")) >= 2, + "expired lease did not consume a bounded retry attempt") + print("PASS: DLQ, guarded replay and expired lease recovery") + + +if __name__ == "__main__": + run(main) diff --git a/student_tests/lab5_fault_injection.py b/student_tests/lab5_fault_injection.py new file mode 100644 index 00000000..73c0be55 --- /dev/null +++ b/student_tests/lab5_fault_injection.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Prove recovery across DB-commit/publish and result-commit/ack crashes.""" + +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] + + +def command(*args, capture=False, environment=None): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, check=True, text=True, + capture_output=capture, env=environment) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def recreate(service, fault_name=None): + environment = os.environ.copy() + environment["FAULT_CRASH_AFTER_OUTBOX_COMMIT"] = "false" + environment["FAULT_CRASH_AFTER_RESULT_COMMIT"] = "false" + if fault_name: + environment[fault_name] = "true" + command("up", "-d", "--no-deps", "--force-recreate", service, + environment=environment) + + +def wait_http(client, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready") + + +def wait_job(attachment_id, expected="ready", seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + status = database_scalar( + "SELECT status FROM attachment_jobs WHERE attachment_id = '" + + attachment_id + "';") + if status == expected: + return + if status == "failed": + raise Failure("fault-injected job reached failed") + time.sleep(1) + raise Failure("fault-injected job did not reach " + expected) + + +def latest_attachment(chat_id, fault_column): + row = database_scalar( + "SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id " + "WHERE a.chat_id = '" + chat_id + "' AND j." + fault_column + + " = true ORDER BY a.created_at DESC LIMIT 1;") + check(bool(row), "fault marker was not persisted") + return row + + +def main(): + client = Client("http://localhost:8080") + wait_http(client) + alice, bob, _, chat = client.bootstrap() + original = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + # Crash exactly after attachment + outbox COMMIT, before the request thread publishes. + recreate("api", "FAULT_CRASH_AFTER_OUTBOX_COMMIT") + wait_http(client) + try: + client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=outbox.png", + 202, actor=alice, raw=original, headers={"Content-Type": "image/png"}, + schema="Attachment") + raise Failure("outbox fault did not terminate the API request") + except Failure as error: + if "outbox fault did not" in str(error): + raise + + outbox_attachment = latest_attachment(chat["id"], "publish_fault_injected") + wait_job(outbox_attachment) + recreate("api") + wait_http(client) + recovered = client.call( + "GET", "/api/v1/attachments/" + outbox_attachment, + actor=bob, schema="Attachment")["json"] + check(recovered["status"] == "ready", "outbox recovery did not publish the job") + check(len(recovered["variants"]) == 3, "outbox recovery did not finish all variants") + + # Crash after S3 effects + result COMMIT, before RabbitMQ consumer ack. + recreate("worker", "FAULT_CRASH_AFTER_RESULT_COMMIT") + queued = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=ack-window.png", + 202, actor=alice, raw=original, headers={"Content-Type": "image/png"}, + schema="Attachment")["json"] + wait_job(queued["id"]) + ready = client.wait_ready(bob, queued, wait_seconds=120) + check(len(ready["variants"]) == 3, "result crash lost variants") + result_marker = database_scalar( + "SELECT result_fault_injected FROM attachment_jobs WHERE attachment_id = '" + + queued["id"] + "';") + check(result_marker == "t", "result commit fault marker was not persisted") + recreate("worker") + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + queues = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", "messages_unacknowledged", capture=True) + if "attachments.process.level4.v1\t0\t0" in queues: + break + time.sleep(1) + variant_count = database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id = '" + + queued["id"] + "';") + check(variant_count == "3", "redelivery duplicated committed variants") + print("PASS: outbox commit/publish crash and result commit/ack crash recovered") + + +if __name__ == "__main__": + run(main) diff --git a/student_tests/lab5_worker_recovery.py b/student_tests/lab5_worker_recovery.py new file mode 100644 index 00000000..dadcf1ac --- /dev/null +++ b/student_tests/lab5_worker_recovery.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Exercise durable queued upload, component restarts and duplicate delivery. + +Run from the repository root against the local compose.yaml: + python student_tests/lab5_worker_recovery.py +The script preserves volumes and leaves the complete stack running. +""" + +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] + + +def command(*args, capture=False): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, check=True, + text=True, capture_output=capture) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def env_value(name): + for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise Failure("missing " + name + " in .env.example") + + +def wait_http_ready(client, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready after restart") + + +def wait_rabbit(seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + result = subprocess.run( + [*COMPOSE, "exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping"], + cwd=ROOT, text=True, capture_output=True) + if result.returncode == 0: + return + time.sleep(2) + raise Failure("RabbitMQ did not become ready after restart") + + +def main(): + client = Client("http://localhost:8080") + command("stop", "worker") + try: + alice, bob, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload did not remain queued") + client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", + 409, actor=bob) + linked = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(linked["attachment_ids"] == [attachment["id"]], + "queued attachment could not be linked") + + command("restart", "api") + command("restart", "rabbitmq") + wait_rabbit() + wait_http_ready(client) + command("start", "worker") + + ready = client.wait_ready(bob, attachment, wait_seconds=120) + check(ready["status"] == "ready", "attachment did not become ready") + downloaded = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + check(downloaded["bytes"] == original, "original bytes changed") + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, + "expected thumbnail, crop and optimized variants") + check((variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160), + "thumbnail dimensions differ from contract") + check((variants["crop"]["width"], variants["crop"]["height"]) == (128, 128), + "crop dimensions differ from contract") + crop = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=crop", + actor=bob) + check(crop["headers"].get_content_type() == "image/jpeg", "crop is not JPEG") + check(jpeg_size(crop["bytes"]) == (128, 128), "crop JPEG has wrong dimensions") + optimized = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=optimized", + actor=bob) + check(optimized["headers"].get_content_type() == "image/webp", "optimized is not WebP") + check(optimized["bytes"][:4] == b"RIFF" and optimized["bytes"][8:12] == b"WEBP", + "optimized payload has no WebP signature") + + job_row = database_scalar( + "SELECT id || '|' || pipeline_version || '|' || attempt || '|' || " + "to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM attachment_jobs WHERE attachment_id = '" + attachment["id"] + "';") + job_id, pipeline_version, attempt, created_at = job_row.split("|") + payload = json.dumps({ + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment["id"], + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created_at, + }, separators=(",", ":")) + command( + "exec", "-T", "rabbitmq", "rabbitmqadmin", + "-u", env_value("RABBITMQ_USER"), + "-p", env_value("RABBITMQ_PASSWORD"), + "publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"), + "routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"), + "payload=" + payload, + 'properties={"delivery_mode":2,"content_type":"application/json"}') + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + queued = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", "messages_unacknowledged", capture=True) + if env_value("ATTACHMENT_JOB_QUEUE") + "\t0\t0" in queued: + break + time.sleep(0.5) + variant_count = database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id = '" + + attachment["id"] + "';") + status = database_scalar( + "SELECT status FROM attachments WHERE id = '" + attachment["id"] + "';") + check(variant_count == "3", "duplicate delivery changed the three variant rows") + check(status == "ready", "duplicate delivery changed terminal ready state") + check(hashlib.sha256(downloaded["bytes"]).hexdigest() == attachment["sha256"], + "original checksum changed") + print("PASS: stopped worker, durable restart recovery and duplicate delivery") + finally: + command("start", "worker") + + +if __name__ == "__main__": + run(main)