4
This commit is contained in:
+48
-19
@@ -1,25 +1,54 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Copy to .env for a local synthetic environment. Never commit .env.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
|
||||
DATABASE_NAME=messenger
|
||||
DATABASE_USER=messenger
|
||||
DATABASE_PASSWORD=replace-with-a-long-random-database-password
|
||||
|
||||
RABBITMQ_USER=messenger
|
||||
RABBITMQ_PASSWORD=replace-with-a-long-random-rabbitmq-password
|
||||
RABBITMQ_VHOST=/
|
||||
ATTACHMENT_JOB_EXCHANGE=attachments.v1
|
||||
ATTACHMENT_JOB_QUEUE=attachments.process.level4.v1
|
||||
ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1
|
||||
ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1
|
||||
ATTACHMENT_DLQ_QUEUE=attachments.failed.v1
|
||||
ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_RETRY_BASE_DELAY=1s
|
||||
JOB_RETRY_MAX_DELAY=30s
|
||||
JOB_TIMEOUT=60s
|
||||
JOB_LEASE_DURATION=90s
|
||||
JOB_LEASE_REAPER_INTERVAL=5s
|
||||
WORKER_MAX_IMAGE_PIXELS=40000000
|
||||
WORKER_MEMORY_BUDGET_BYTES=536870912
|
||||
WORKER_MEMORY_WAIT=2s
|
||||
|
||||
# Fault injection is opt-in and intended only for an isolated test stand.
|
||||
FAULT_CRASH_AFTER_OUTBOX_COMMIT=false
|
||||
FAULT_CRASH_AFTER_RESULT_COMMIT=false
|
||||
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-api-secret
|
||||
MINIO_ROOT_USER=minio-root
|
||||
MINIO_ROOT_PASSWORD=replace-with-a-different-long-random-root-secret
|
||||
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
MAX_CONCURRENT_UPLOADS=3
|
||||
UPLOAD_PERMIT_WAIT_MILLIS=10000
|
||||
|
||||
# Staging objects are deleted only after the final result is committed.
|
||||
# A separate lifecycle/cleanup policy may remove abandoned staging objects
|
||||
# after this retention interval; it must never remove active jobs.
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
ATTACHMENT_CLEANUP_GRACE_SECONDS=3600
|
||||
ATTACHMENT_CLEANUP_DRY_RUN=false
|
||||
ATTACHMENT_RECONCILE_INTERVAL_MILLIS=600000
|
||||
ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS=60000
|
||||
|
||||
@@ -15,6 +15,7 @@ data/
|
||||
backups/
|
||||
# Runtime/build artifacts
|
||||
.venv/
|
||||
.gradle/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
node_modules/
|
||||
|
||||
@@ -1,37 +1,71 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Целевая оценка: 4
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Библиотеки: Spring AMQP, RabbitMQ Java client, Thumbnailator, webp-imageio, MinIO SDK, PostgreSQL/Flyway
|
||||
- Помощь ИИ: проектирование pipeline, реализация и подготовка тестов; результат проверен сборкой, unit- и black-box-тестами
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Desktop/Engine с Compose v2, Java 23 и Python 3.9+.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# Заменить все replace-with-* уникальными локальными значениями.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8080
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных выполняется только явной командой оператора с `--volumes`.
|
||||
|
||||
## Доказательства критериев
|
||||
## Pipeline и гарантии
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет неизменённый original в приватный `staging/<attachment_id>/original`, затем одной транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`. Запись job является transactional outbox. Фоновый publisher публикует persistent JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а HTTP `202` означает, что staging и намерение обработки уже устойчивы.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Отдельный `worker` захватывает job с lease, читает доверенный staging key из PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только `schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют. Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже `ready` job — no-op.
|
||||
|
||||
## Эксперименты и отказы
|
||||
Обязательные варианты pipeline version 1:
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
| Вариант | Правило | Формат/качество |
|
||||
| --- | --- | --- |
|
||||
| `original` | bytes и SHA-256 совпадают с upload | исходный MIME |
|
||||
| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality 0.85 |
|
||||
| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality 0.88 |
|
||||
| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality 0.82 |
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных metadata.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Ограничения worker и retry
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Upload: `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000`.
|
||||
- Worker повторно проверяет header/dimensions до полного decode и применяет тот же pixel cap.
|
||||
- Оценка памяти на job резервируется общим честным semaphore; бюджет `WORKER_MEMORY_BUDGET_BYTES=536870912`, ожидание `WORKER_MEMORY_WAIT=2s`. Compose дополнительно ограничивает worker `1024m`, JVM использует до 70% container RAM.
|
||||
- Обработка ограничена `JOB_TIMEOUT=60s`; просроченный task теряет lease и не может зафиксировать результат.
|
||||
- `WORKER_CONCURRENCY=2`, `WORKER_PREFETCH=1`; lease 90s, reaper каждые 5s.
|
||||
- Не более `JOB_MAX_ATTEMPTS=5`. Временные ошибки получают exponential backoff с jitter 50–100% и cap 30s. Ошибки decode/format/size постоянные и сразу становятся безопасным `failed/error_code`; exhausted retry — `retry_exhausted`.
|
||||
- Permanent malformed broker delivery и исчерпанная работа dead-lettered в durable `attachments.failed.v1`. Реестр failed остаётся в PostgreSQL.
|
||||
|
||||
Операторский replay выполняется `scripts/replay-failed.ps1 -JobId <uuid>` или `scripts/replay-failed.sh <uuid>`. Одна транзакция блокирует одновременно failed job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние. Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`.
|
||||
|
||||
После безопасного COMMIT результата worker best-effort удаляет staging. `StagingCleaner` удаляет только неактивные объекты старше `STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён.
|
||||
|
||||
## Fault injection и проверки
|
||||
|
||||
| Критерий | Команда | Результат |
|
||||
| --- | --- | --- |
|
||||
| Unit tests | `gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' --tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'` | `BUILD SUCCESSFUL` |
|
||||
| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS |
|
||||
| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:8080` | PASS |
|
||||
| Worker stop, broker/API restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows |
|
||||
| COMMIT→publish и result COMMIT→ack crash | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи автоматически дошли до ready |
|
||||
| DLQ, replay, lease | `python -X utf8 student_tests/lab5_dlq_replay_lease.py` | PASS; DLQ +1, одна audit-запись, ready не replayed, expired lease восстановлен |
|
||||
|
||||
Fault flags выключены по умолчанию. `FAULT_CRASH_AFTER_OUTBOX_COMMIT=true` один раз завершает API через `Runtime.halt(87)` после DB COMMIT до publish. После restart republisher находит `published=false`, и job достигает ready. `FAULT_CRASH_AFTER_RESULT_COMMIT=true` один раз завершает worker через `Runtime.halt(86)` после атомарного ready до consumer ack; RabbitMQ redelivery проверяет ready и завершает ту же job без второго набора вариантов.
|
||||
|
||||
Подробный журнал: `evidence/lab5-level4-2026-11-19.md`.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Это уровень 4, не исследовательский трек 5: производительность для матрицы concurrency/prefetch не измерялась, multi-version reorder не исследовался. RabbitMQ, PostgreSQL и MinIO развёрнуты по одному экземпляру; volumes переживают container restart, но не потерю host/disk. At-least-once + идемпотентный эффект не является exactly-once доставкой.
|
||||
|
||||
@@ -24,6 +24,7 @@ dependencies {
|
||||
implementation 'org.springframework.boot:spring-boot-starter-validation'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-amqp'
|
||||
implementation 'org.springframework.boot:spring-boot-flyway'
|
||||
implementation 'org.flywaydb:flyway-core'
|
||||
implementation 'org.flywaydb:flyway-database-postgresql'
|
||||
@@ -31,6 +32,8 @@ dependencies {
|
||||
implementation 'org.bouncycastle:bcprov-jdk18on:1.84'
|
||||
implementation 'org.apache.commons:commons-pool2'
|
||||
implementation 'io.minio:minio:8.6.0'
|
||||
implementation 'net.coobird:thumbnailator:0.4.20'
|
||||
implementation 'org.sejda.imageio:webp-imageio:0.1.6'
|
||||
runtimeOnly 'org.postgresql:postgresql'
|
||||
|
||||
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
|
||||
|
||||
+98
-11
@@ -1,4 +1,4 @@
|
||||
name: hla-lab4
|
||||
name: hla-lab5
|
||||
|
||||
x-api-environment: &api-environment
|
||||
APP_HOST: 0.0.0.0
|
||||
@@ -30,6 +30,29 @@ x-api-environment: &api-environment
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
|
||||
UPLOAD_PERMIT_WAIT_MILLIS: ${UPLOAD_PERMIT_WAIT_MILLIS:-10000}
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_PORT: 5672
|
||||
RABBITMQ_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
|
||||
RABBITMQ_VHOST: ${RABBITMQ_VHOST:-/}
|
||||
ATTACHMENT_JOB_EXCHANGE: ${ATTACHMENT_JOB_EXCHANGE:-attachments.v1}
|
||||
ATTACHMENT_JOB_QUEUE: ${ATTACHMENT_JOB_QUEUE:-attachments.process.level4.v1}
|
||||
ATTACHMENT_JOB_ROUTING_KEY: ${ATTACHMENT_JOB_ROUTING_KEY:-attachment.process.v1}
|
||||
ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1}
|
||||
ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1}
|
||||
ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_RETRY_BASE_DELAY: ${JOB_RETRY_BASE_DELAY:-1s}
|
||||
JOB_RETRY_MAX_DELAY: ${JOB_RETRY_MAX_DELAY:-30s}
|
||||
JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
|
||||
JOB_LEASE_DURATION: ${JOB_LEASE_DURATION:-90s}
|
||||
JOB_LEASE_REAPER_INTERVAL: ${JOB_LEASE_REAPER_INTERVAL:-5s}
|
||||
WORKER_MAX_IMAGE_PIXELS: ${WORKER_MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_MEMORY_BUDGET_BYTES: ${WORKER_MEMORY_BUDGET_BYTES:-536870912}
|
||||
WORKER_MEMORY_WAIT: ${WORKER_MEMORY_WAIT:-2s}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600}
|
||||
ATTACHMENT_CLEANUP_DRY_RUN: ${ATTACHMENT_CLEANUP_DRY_RUN:-false}
|
||||
ATTACHMENT_RECONCILE_INTERVAL_MILLIS: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:-600000}
|
||||
@@ -69,6 +92,26 @@ services:
|
||||
mem_limit: 256m
|
||||
networks: [backend]
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.4-management-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
|
||||
RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/}
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
||||
volumes:
|
||||
- rabbitmq-data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
start_period: 60s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 1024m
|
||||
networks: [backend]
|
||||
|
||||
minio:
|
||||
image: bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd0e172c4cc0870f4bdc3142018e2a37be9acf04d68f386600daad427e0cab
|
||||
environment:
|
||||
@@ -103,13 +146,24 @@ services:
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing "local/$$S3_BUCKET"
|
||||
mc anonymous set none "local/$$S3_BUCKET"
|
||||
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy/minio-api-policy.json > /tmp/minio-api-policy.json
|
||||
mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" || true
|
||||
mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json || true
|
||||
mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID"
|
||||
attempt=0
|
||||
until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" \
|
||||
&& mc mb --ignore-existing "local/$$S3_BUCKET" \
|
||||
&& mc anonymous set none "local/$$S3_BUCKET" \
|
||||
&& (mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" \
|
||||
|| mc admin user info local "$$S3_ACCESS_KEY_ID") \
|
||||
&& (mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json \
|
||||
|| mc admin policy info local messenger-attachments-rw) \
|
||||
&& mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID"
|
||||
do
|
||||
attempt=$$((attempt + 1))
|
||||
if [ "$$attempt" -ge 30 ]; then
|
||||
echo "MinIO bootstrap did not complete after 30 attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
@@ -144,38 +198,45 @@ services:
|
||||
|
||||
# One-shot Flyway runner. It is invoked explicitly before API replicas.
|
||||
migrate:
|
||||
image: hla-lab4-api:local
|
||||
image: hla-lab5-app:local
|
||||
build:
|
||||
context: .
|
||||
environment:
|
||||
<<: *api-environment
|
||||
FLYWAY_ENABLED: "true"
|
||||
MIGRATE_ONLY: "true"
|
||||
MESSENGER_ROLE: migration
|
||||
SPRING_MAIN_WEB_APPLICATION_TYPE: none
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
networks: [backend]
|
||||
|
||||
api:
|
||||
image: hla-lab4-api:local
|
||||
image: hla-lab5-app:local
|
||||
build:
|
||||
context: .
|
||||
environment:
|
||||
<<: *api-environment
|
||||
FLYWAY_ENABLED: "false"
|
||||
MESSENGER_ROLE: api
|
||||
FAULT_CRASH_AFTER_OUTBOX_COMMIT: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:-false}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"]
|
||||
interval: 2s
|
||||
@@ -189,6 +250,31 @@ services:
|
||||
mem_limit: 768m
|
||||
networks: [backend]
|
||||
|
||||
worker:
|
||||
image: hla-lab5-app:local
|
||||
environment:
|
||||
<<: *api-environment
|
||||
FLYWAY_ENABLED: "false"
|
||||
MESSENGER_ROLE: worker
|
||||
SPRING_MAIN_WEB_APPLICATION_TYPE: none
|
||||
JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70
|
||||
FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
init: true
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
cpus: 1.0
|
||||
mem_limit: 1024m
|
||||
networks: [backend]
|
||||
|
||||
nginx:
|
||||
image: nginx:1.27.4-bookworm
|
||||
ports:
|
||||
@@ -217,3 +303,4 @@ volumes:
|
||||
postgres-data:
|
||||
redis-data:
|
||||
minio-data:
|
||||
rabbitmq-data:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# Lab5 verification — 2026-11-18
|
||||
|
||||
Среда: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23, Spring Boot 4.1.1, RabbitMQ 4.1.4.
|
||||
|
||||
## Статические и unit-проверки
|
||||
|
||||
```text
|
||||
docker compose --env-file .env.example config --quiet
|
||||
PASS (exit 0)
|
||||
|
||||
gradlew test --tests '*AttachmentServiceTest' --tests '*ThumbnailProcessorTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'
|
||||
BUILD SUCCESSFUL
|
||||
```
|
||||
|
||||
## Публичный сценарий
|
||||
|
||||
```text
|
||||
python tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
```
|
||||
|
||||
В момент проверки отдельные services `api`, `worker`, `rabbitmq`, `postgres`, `redis`, `minio` и `nginx` были запущены; RabbitMQ queue `attachments.process.v1` имела `durable=true`.
|
||||
|
||||
## Остановка worker, restart и redelivery
|
||||
|
||||
```text
|
||||
python student_tests/lab5_worker_recovery.py
|
||||
PASS: stopped worker, durable restart recovery and duplicate delivery
|
||||
```
|
||||
|
||||
Сценарий теста:
|
||||
|
||||
1. останавливает worker;
|
||||
2. получает upload `202 queued`, проверяет `409` content и привязку к сообщению;
|
||||
3. перезапускает API и RabbitMQ без удаления volumes;
|
||||
4. запускает worker и ждёт `ready`;
|
||||
5. сверяет original bytes и единственный thumbnail;
|
||||
6. публикует тот же `job_id` повторно;
|
||||
7. проверяет terminal `ready` и ровно одну запись thumbnail.
|
||||
|
||||
Синтетические пароли и токены тест не печатает и в evidence не сохраняет.
|
||||
@@ -0,0 +1,56 @@
|
||||
# Lab5 level 4 verification — 2026-11-19
|
||||
|
||||
Среда: Windows, Docker Compose, Java 23, Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, MinIO.
|
||||
|
||||
## Статические и unit-проверки
|
||||
|
||||
```text
|
||||
python -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
docker compose --env-file .env.example config --quiet
|
||||
exit 0
|
||||
|
||||
gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' \
|
||||
--tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' \
|
||||
--tests '*StagingCleanerTest'
|
||||
BUILD SUCCESSFUL
|
||||
```
|
||||
|
||||
`ImageVariantProcessorTest` проверяет реальные JPEG/WebP signatures, ImageIO decode и размеры thumbnail/crop/optimized. `RetryPolicyTest` проверяет bounded exponential backoff и jitter.
|
||||
|
||||
## API и recovery
|
||||
|
||||
```text
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
python -X utf8 student_tests/lab5_worker_recovery.py
|
||||
PASS: stopped worker, durable restart recovery and duplicate delivery
|
||||
```
|
||||
|
||||
Recovery test останавливает worker, получает `202 queued`, перезапускает API и RabbitMQ без удаления volumes, запускает worker и сверяет original SHA-256, thumbnail 256×160, crop 128×128, WebP signature. Повторная публикация того же `job_id` оставляет `ready` и ровно три variant rows.
|
||||
|
||||
## Доказанные окна отказа
|
||||
|
||||
```text
|
||||
python -X utf8 student_tests/lab5_fault_injection.py
|
||||
PASS: outbox commit/publish crash and result commit/ack crash recovered
|
||||
```
|
||||
|
||||
1. API завершается после COMMIT attachment/outbox до publish. Перезапущенный API republisher публикует сохранённое намерение; новый upload не создаётся вручную.
|
||||
2. Worker завершается после записи объектов и atomic ready COMMIT до ack. Redelivery завершает тот же effect; variant rows остаются уникальными.
|
||||
|
||||
## DLQ, replay и lease
|
||||
|
||||
```text
|
||||
python -X utf8 student_tests/lab5_dlq_replay_lease.py
|
||||
PASS: DLQ, guarded replay and expired lease recovery
|
||||
```
|
||||
|
||||
- malformed persistent JSON был permanent failure и увеличил `attachments.failed.v1` ровно на 1;
|
||||
- failed job replayed один раз, audit count = 1; повтор в queued/ready дал no-op;
|
||||
- после replay получены те же три variant names без второго набора;
|
||||
- искусственно истёкший processing lease был requeued, attempt увеличился, job дошёл до ready.
|
||||
|
||||
Секреты, session tokens и contents сообщений в evidence не сохранялись. Docker volumes в проверках не удалялись.
|
||||
@@ -19,7 +19,8 @@
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__/attachments/*"
|
||||
"arn:aws:s3:::__S3_BUCKET__/attachments/*",
|
||||
"arn:aws:s3:::__S3_BUCKET__/staging/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobId,
|
||||
[string]$RequestedBy = $env:USERNAME,
|
||||
[string]$EnvFile = ".env"
|
||||
)
|
||||
|
||||
$parsedJobId = [guid]::Empty
|
||||
if (-not [guid]::TryParse($JobId, [ref]$parsedJobId)) {
|
||||
throw "JobId must be a UUID"
|
||||
}
|
||||
if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') {
|
||||
throw "RequestedBy contains unsupported characters"
|
||||
}
|
||||
|
||||
$sql = @"
|
||||
WITH candidate AS (
|
||||
SELECT j.id, j.attachment_id, j.attempt
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = '$parsedJobId'::uuid
|
||||
AND j.status = 'failed' AND a.status = 'failed'
|
||||
FOR UPDATE OF j, a
|
||||
), audited AS (
|
||||
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
|
||||
SELECT id, '$RequestedBy', attempt FROM candidate
|
||||
RETURNING job_id
|
||||
), reset_job AS (
|
||||
UPDATE attachment_jobs j
|
||||
SET status = 'queued', attempt = 1, published = false,
|
||||
last_error_code = NULL, next_attempt_at = clock_timestamp(),
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c, audited a
|
||||
WHERE j.id = c.id AND a.job_id = c.id
|
||||
RETURNING j.attachment_id
|
||||
), reset_attachment AS (
|
||||
UPDATE attachments a
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
FROM reset_job r
|
||||
WHERE a.id = r.attachment_id AND a.status = 'failed'
|
||||
RETURNING a.id
|
||||
)
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment)
|
||||
THEN 'replayed' ELSE 'no-op' END;
|
||||
"@
|
||||
|
||||
$result = docker compose --env-file $EnvFile exec -T postgres `
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Replay transaction failed"
|
||||
}
|
||||
|
||||
if (($result | Select-Object -Last 1).Trim() -eq "replayed") {
|
||||
Write-Output "Replay requested for failed job $parsedJobId by $RequestedBy"
|
||||
} else {
|
||||
Write-Output "No-op: job $parsedJobId is missing, not failed, or already recovered"
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
job_id="${1:?usage: replay-failed.sh JOB_UUID [REQUESTED_BY] [ENV_FILE]}"
|
||||
requested_by="${2:-operator}"
|
||||
env_file="${3:-.env}"
|
||||
|
||||
case "$job_id" in
|
||||
????????-????-????-????-????????????) ;;
|
||||
*) echo "JobId must be a UUID" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$requested_by" in
|
||||
*[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
sql="WITH candidate AS (
|
||||
SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed'
|
||||
FOR UPDATE OF j, a
|
||||
), audited AS (
|
||||
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
|
||||
SELECT id, '$requested_by', attempt FROM candidate RETURNING job_id
|
||||
), reset_job AS (
|
||||
UPDATE attachment_jobs j SET status = 'queued', attempt = 1, published = false,
|
||||
last_error_code = NULL, next_attempt_at = clock_timestamp(),
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c, audited a WHERE j.id = c.id AND a.job_id = c.id
|
||||
RETURNING j.attachment_id
|
||||
), reset_attachment AS (
|
||||
UPDATE attachments a SET status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
FROM reset_job r WHERE a.id = r.attachment_id AND a.status = 'failed'
|
||||
RETURNING a.id
|
||||
)
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment)
|
||||
THEN 'replayed' ELSE 'no-op' END;"
|
||||
|
||||
result="$(docker compose --env-file "$env_file" exec -T postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")"
|
||||
if [ "$result" = "replayed" ]; then
|
||||
echo "Replay requested for failed job $job_id by $requested_by"
|
||||
else
|
||||
echo "No-op: job $job_id is missing, not failed, or already recovered"
|
||||
fi
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.example.mtusi_messenger.config;
|
||||
|
||||
import org.springframework.amqp.core.Binding;
|
||||
import org.springframework.amqp.core.BindingBuilder;
|
||||
import org.springframework.amqp.core.DirectExchange;
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class RabbitJobConfig {
|
||||
@Bean
|
||||
DirectExchange attachmentExchange(@Value("${messenger.jobs.exchange}") String name) {
|
||||
return new DirectExchange(name, true, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
Queue attachmentQueue(
|
||||
@Value("${messenger.jobs.queue}") String name,
|
||||
@Value("${messenger.jobs.dead-letter-exchange}") String deadLetterExchange,
|
||||
@Value("${messenger.jobs.dead-letter-routing-key}") String deadLetterRoutingKey) {
|
||||
return QueueBuilder.durable(name)
|
||||
.withArgument("x-dead-letter-exchange", deadLetterExchange)
|
||||
.withArgument("x-dead-letter-routing-key", deadLetterRoutingKey)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
DirectExchange attachmentDeadLetterExchange(
|
||||
@Value("${messenger.jobs.dead-letter-exchange}") String name) {
|
||||
return new DirectExchange(name, true, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
Queue attachmentDeadLetterQueue(@Value("${messenger.jobs.dead-letter-queue}") String name) {
|
||||
return QueueBuilder.durable(name).build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
Binding attachmentBinding(
|
||||
@Qualifier("attachmentQueue") Queue attachmentQueue,
|
||||
@Qualifier("attachmentExchange") DirectExchange attachmentExchange,
|
||||
@Value("${messenger.jobs.routing-key}") String routingKey) {
|
||||
return BindingBuilder.bind(attachmentQueue).to(attachmentExchange).with(routingKey);
|
||||
}
|
||||
|
||||
@Bean
|
||||
Binding attachmentDeadLetterBinding(
|
||||
@Qualifier("attachmentDeadLetterQueue") Queue queue,
|
||||
@Qualifier("attachmentDeadLetterExchange") DirectExchange exchange,
|
||||
@Value("${messenger.jobs.dead-letter-routing-key}") String routingKey) {
|
||||
return BindingBuilder.bind(queue).to(exchange).with(routingKey);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.example.mtusi_messenger.controller;
|
||||
|
||||
import com.example.mtusi_messenger.dto.AttachmentResponse;
|
||||
import com.example.mtusi_messenger.exception.ApiExceptions;
|
||||
import com.example.mtusi_messenger.model.Attachment;
|
||||
import com.example.mtusi_messenger.security.SessionPrincipal;
|
||||
import com.example.mtusi_messenger.service.AttachmentService;
|
||||
@@ -47,7 +46,7 @@ public class AttachmentController {
|
||||
) throws IOException {
|
||||
Attachment attachment = attachments.upload(parse(chatId), principal.user().id(), filename,
|
||||
contentType, request.getContentLengthLong(), request.getInputStream());
|
||||
return ResponseEntity.status(HttpStatus.CREATED).body(AttachmentResponse.from(attachment));
|
||||
return ResponseEntity.status(HttpStatus.ACCEPTED).body(AttachmentResponse.from(attachment));
|
||||
}
|
||||
|
||||
@GetMapping("/attachments/{attachmentId}")
|
||||
@@ -64,15 +63,13 @@ public class AttachmentController {
|
||||
@PathVariable String attachmentId,
|
||||
@RequestParam(defaultValue = "original") String variant
|
||||
) {
|
||||
if (!"original".equals(variant)) {
|
||||
throw ApiExceptions.invalid("Unsupported attachment variant");
|
||||
}
|
||||
AttachmentService.Download download = attachments.download(parse(attachmentId), principal.user().id());
|
||||
AttachmentService.Download download = attachments.download(
|
||||
parse(attachmentId), principal.user().id(), variant);
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.parseMediaType(download.attachment().contentType()))
|
||||
.contentType(MediaType.parseMediaType(download.contentType()))
|
||||
.contentLength(download.bytes().length)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.inline()
|
||||
.filename(download.attachment().filename(), StandardCharsets.UTF_8).build().toString())
|
||||
.filename(download.filename(), StandardCharsets.UTF_8).build().toString())
|
||||
.body(download.bytes());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ public record AttachmentResponse(UUID id,
|
||||
long size_bytes,
|
||||
String sha256,
|
||||
String status,
|
||||
List<Object> variants,
|
||||
List<AttachmentVariantResponse> variants,
|
||||
String error_code,
|
||||
Instant created_at,
|
||||
Instant updated_at) {
|
||||
@@ -27,9 +27,9 @@ public record AttachmentResponse(UUID id,
|
||||
attachment.contentType(),
|
||||
attachment.sizeBytes(),
|
||||
attachment.sha256(),
|
||||
"ready",
|
||||
List.of(),
|
||||
null,
|
||||
attachment.status(),
|
||||
attachment.variants().stream().map(AttachmentVariantResponse::from).toList(),
|
||||
attachment.errorCode(),
|
||||
attachment.createdAt(),
|
||||
attachment.updatedAt());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.example.mtusi_messenger.dto;
|
||||
|
||||
import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
|
||||
public record AttachmentVariantResponse(String name,
|
||||
String content_type,
|
||||
long size_bytes,
|
||||
int width,
|
||||
int height,
|
||||
String sha256) {
|
||||
public static AttachmentVariantResponse from(AttachmentVariant variant) {
|
||||
return new AttachmentVariantResponse(
|
||||
variant.name(), variant.contentType(), variant.sizeBytes(),
|
||||
variant.width(), variant.height(), variant.sha256());
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.example.mtusi_messenger.model;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public record Attachment(UUID id,
|
||||
@@ -11,6 +12,27 @@ public record Attachment(UUID id,
|
||||
long sizeBytes,
|
||||
String sha256,
|
||||
String objectKey,
|
||||
String stagingKey,
|
||||
String status,
|
||||
List<AttachmentVariant> variants,
|
||||
String errorCode,
|
||||
Instant createdAt,
|
||||
Instant updatedAt) {
|
||||
public Attachment {
|
||||
variants = variants == null ? List.of() : List.copyOf(variants);
|
||||
}
|
||||
|
||||
public Attachment(UUID id,
|
||||
UUID chatId,
|
||||
UUID ownerId,
|
||||
String filename,
|
||||
String contentType,
|
||||
long sizeBytes,
|
||||
String sha256,
|
||||
String objectKey,
|
||||
Instant createdAt,
|
||||
Instant updatedAt) {
|
||||
this(id, chatId, ownerId, filename, contentType, sizeBytes, sha256, objectKey,
|
||||
null, "ready", List.of(), null, createdAt, updatedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.example.mtusi_messenger.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.UUID;
|
||||
|
||||
public record AttachmentJob(
|
||||
@JsonProperty("schema_version") int schemaVersion,
|
||||
@JsonProperty("job_id") UUID jobId,
|
||||
String type,
|
||||
@JsonProperty("attachment_id") UUID attachmentId,
|
||||
@JsonProperty("pipeline_version") int pipelineVersion,
|
||||
int attempt,
|
||||
@JsonProperty("created_at") Instant createdAt) {
|
||||
|
||||
public static final int SCHEMA_VERSION = 1;
|
||||
public static final int PIPELINE_VERSION = 1;
|
||||
public static final String TYPE = "attachment.process.v1";
|
||||
|
||||
public AttachmentJob {
|
||||
if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type)
|
||||
|| pipelineVersion < 1 || attempt < 1) {
|
||||
throw new IllegalArgumentException("Unsupported attachment job message");
|
||||
}
|
||||
}
|
||||
|
||||
public static AttachmentJob first(UUID jobId, UUID attachmentId, Instant createdAt) {
|
||||
return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId,
|
||||
PIPELINE_VERSION, 1, createdAt);
|
||||
}
|
||||
|
||||
public AttachmentJob nextAttempt(int nextAttempt) {
|
||||
return new AttachmentJob(schemaVersion, jobId, type, attachmentId,
|
||||
pipelineVersion, nextAttempt, createdAt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.example.mtusi_messenger.model;
|
||||
|
||||
public record AttachmentVariant(String name,
|
||||
String contentType,
|
||||
long sizeBytes,
|
||||
int width,
|
||||
int height,
|
||||
String sha256,
|
||||
String objectKey) {
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package com.example.mtusi_messenger.queue;
|
||||
|
||||
import com.example.mtusi_messenger.model.AttachmentJob;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.core.MessageDeliveryMode;
|
||||
import org.springframework.amqp.core.MessageProperties;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
@Component
|
||||
public class AttachmentJobPublisher {
|
||||
private final RabbitTemplate rabbit;
|
||||
private final ObjectMapper mapper;
|
||||
private final String exchange;
|
||||
private final String routingKey;
|
||||
private final long confirmTimeoutMillis;
|
||||
|
||||
public AttachmentJobPublisher(
|
||||
RabbitTemplate rabbit,
|
||||
ObjectMapper mapper,
|
||||
@Value("${messenger.jobs.exchange}") String exchange,
|
||||
@Value("${messenger.jobs.routing-key}") String routingKey,
|
||||
@Value("${messenger.jobs.publish-confirm-timeout:5s}") Duration confirmTimeout) {
|
||||
this.rabbit = rabbit;
|
||||
this.mapper = mapper;
|
||||
this.exchange = exchange;
|
||||
this.routingKey = routingKey;
|
||||
this.confirmTimeoutMillis = confirmTimeout.toMillis();
|
||||
this.rabbit.setMandatory(true);
|
||||
}
|
||||
|
||||
public void publish(AttachmentJob job) {
|
||||
try {
|
||||
byte[] body = mapper.writeValueAsBytes(job);
|
||||
MessageProperties properties = new MessageProperties();
|
||||
properties.setContentType(MessageProperties.CONTENT_TYPE_JSON);
|
||||
properties.setContentEncoding("UTF-8");
|
||||
properties.setDeliveryMode(MessageDeliveryMode.PERSISTENT);
|
||||
properties.setMessageId(job.jobId().toString());
|
||||
properties.setHeader("schema_version", job.schemaVersion());
|
||||
Message message = new Message(body, properties);
|
||||
|
||||
rabbit.invoke(operations -> {
|
||||
operations.send(exchange, routingKey, message);
|
||||
operations.waitForConfirmsOrDie(confirmTimeoutMillis);
|
||||
return null;
|
||||
});
|
||||
} catch (JsonProcessingException error) {
|
||||
throw new JobPublishException("Cannot encode attachment job", error);
|
||||
} catch (RuntimeException error) {
|
||||
throw new JobPublishException("RabbitMQ did not confirm attachment job", error);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class JobPublishException extends RuntimeException {
|
||||
public JobPublishException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.example.mtusi_messenger.queue;
|
||||
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.time.Clock;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
|
||||
public class ExpiredLeaseReaper {
|
||||
private static final Logger log = LoggerFactory.getLogger(ExpiredLeaseReaper.class);
|
||||
private final AttachmentRepository attachments;
|
||||
private final RetryPolicy retryPolicy;
|
||||
private final TransactionTemplate transaction;
|
||||
private final int maxAttempts;
|
||||
private final Clock clock = Clock.systemUTC();
|
||||
|
||||
public ExpiredLeaseReaper(
|
||||
AttachmentRepository attachments,
|
||||
RetryPolicy retryPolicy,
|
||||
TransactionTemplate transaction,
|
||||
@Value("${messenger.jobs.max-attempts:5}") int maxAttempts) {
|
||||
this.attachments = attachments;
|
||||
this.retryPolicy = retryPolicy;
|
||||
this.transaction = transaction;
|
||||
this.maxAttempts = maxAttempts;
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${messenger.jobs.lease-reaper-interval:5s}")
|
||||
public void recoverExpired() {
|
||||
for (var job : attachments.findExpiredJobs(50)) {
|
||||
try {
|
||||
transaction.executeWithoutResult(status -> {
|
||||
if (job.attempt() >= maxAttempts) {
|
||||
attachments.failExpired(job.jobId(), job.attachmentId(), "retry_exhausted");
|
||||
} else {
|
||||
int nextAttempt = job.attempt() + 1;
|
||||
attachments.requeueExpired(
|
||||
job.jobId(), job.attachmentId(), nextAttempt, "processing_timeout",
|
||||
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)));
|
||||
}
|
||||
});
|
||||
} catch (RuntimeException error) {
|
||||
log.warn("Expired attachment lease recovery postponed for job {}", job.jobId());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.example.mtusi_messenger.queue;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
|
||||
@Component
|
||||
public class RetryPolicy {
|
||||
private final long baseDelayMillis;
|
||||
private final long maxDelayMillis;
|
||||
|
||||
public RetryPolicy(
|
||||
@Value("${messenger.jobs.retry-base-delay:1s}") Duration baseDelay,
|
||||
@Value("${messenger.jobs.retry-max-delay:30s}") Duration maxDelay) {
|
||||
if (baseDelay.isNegative() || baseDelay.isZero() || maxDelay.compareTo(baseDelay) < 0) {
|
||||
throw new IllegalArgumentException("Invalid retry delay configuration");
|
||||
}
|
||||
this.baseDelayMillis = baseDelay.toMillis();
|
||||
this.maxDelayMillis = maxDelay.toMillis();
|
||||
}
|
||||
|
||||
public Duration delayForAttempt(int nextAttempt) {
|
||||
int exponent = Math.max(0, Math.min(30, nextAttempt - 2));
|
||||
long exponential;
|
||||
try {
|
||||
exponential = Math.multiplyExact(baseDelayMillis, 1L << exponent);
|
||||
} catch (ArithmeticException overflow) {
|
||||
exponential = maxDelayMillis;
|
||||
}
|
||||
long ceiling = Math.min(maxDelayMillis, exponential);
|
||||
long floor = Math.max(1, ceiling / 2);
|
||||
long jittered = floor == ceiling
|
||||
? ceiling
|
||||
: ThreadLocalRandom.current().nextLong(floor, ceiling + 1);
|
||||
return Duration.ofMillis(jittered);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.example.mtusi_messenger.queue;
|
||||
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
|
||||
public class UnpublishedJobRepublisher {
|
||||
private static final Logger log = LoggerFactory.getLogger(UnpublishedJobRepublisher.class);
|
||||
private final AttachmentRepository attachments;
|
||||
private final AttachmentJobPublisher publisher;
|
||||
|
||||
public UnpublishedJobRepublisher(AttachmentRepository attachments, AttachmentJobPublisher publisher) {
|
||||
this.attachments = attachments;
|
||||
this.publisher = publisher;
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${messenger.jobs.republish-interval:5s}")
|
||||
public void publishPending() {
|
||||
for (var job : attachments.findUnpublishedJobs(50)) {
|
||||
try {
|
||||
publisher.publish(job);
|
||||
attachments.markPublished(job.jobId());
|
||||
} catch (RuntimeException error) {
|
||||
log.warn("Attachment job publication postponed for job {}", job.jobId());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,37 @@
|
||||
package com.example.mtusi_messenger.repository;
|
||||
|
||||
import com.example.mtusi_messenger.model.Attachment;
|
||||
import com.example.mtusi_messenger.model.AttachmentJob;
|
||||
import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.Set;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
public interface AttachmentRepository {
|
||||
Attachment save(Attachment attachment);
|
||||
Attachment saveQueued(Attachment attachment, AttachmentJob job);
|
||||
Optional<Attachment> findById(UUID id);
|
||||
List<Attachment> findAllByIdsInChat(UUID chatId, List<UUID> ids);
|
||||
Set<String> findAllObjectKeys();
|
||||
Set<String> findActiveStagingKeys();
|
||||
void markPublished(UUID jobId);
|
||||
boolean markPublishFaultInjected(UUID jobId);
|
||||
List<AttachmentJob> findUnpublishedJobs(int limit);
|
||||
int currentAttempt(UUID jobId);
|
||||
boolean claim(UUID jobId, UUID attachmentId, int attempt, UUID leaseToken, Duration leaseDuration);
|
||||
boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken,
|
||||
int nextAttempt, String errorCode, Instant nextAttemptAt);
|
||||
CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion,
|
||||
List<AttachmentVariant> variants, boolean injectCrashAfterCommit);
|
||||
boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode);
|
||||
List<AttachmentJob> findExpiredJobs(int limit);
|
||||
boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt,
|
||||
String errorCode, Instant nextAttemptAt);
|
||||
boolean failExpired(UUID jobId, UUID attachmentId, String errorCode);
|
||||
|
||||
record CompletionResult(boolean completed, boolean crashAfterCommit) {
|
||||
}
|
||||
}
|
||||
|
||||
+268
-12
@@ -1,22 +1,28 @@
|
||||
package com.example.mtusi_messenger.repository;
|
||||
|
||||
import com.example.mtusi_messenger.model.Attachment;
|
||||
import com.example.mtusi_messenger.model.AttachmentJob;
|
||||
import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.Set;
|
||||
import java.util.HashSet;
|
||||
import java.util.UUID;
|
||||
|
||||
@Repository
|
||||
public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
private static final String COLUMNS = """
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, created_at, updated_at
|
||||
sha256, object_key, staging_key, status, error_code, created_at, updated_at
|
||||
""";
|
||||
private final JdbcTemplate jdbc;
|
||||
|
||||
@@ -25,16 +31,22 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Attachment save(Attachment attachment) {
|
||||
public Attachment saveQueued(Attachment attachment, AttachmentJob job) {
|
||||
jdbc.update("""
|
||||
INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, status, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'ready', ?, ?)
|
||||
sha256, object_key, staging_key, status, error_code, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?)
|
||||
""", attachment.id(), attachment.chatId(), attachment.ownerId(),
|
||||
attachment.filename(), attachment.contentType(), attachment.sizeBytes(),
|
||||
attachment.sha256(), attachment.objectKey(), Timestamp.from(attachment.createdAt()),
|
||||
Timestamp.from(attachment.updatedAt()));
|
||||
attachment.sha256(), attachment.objectKey(), attachment.stagingKey(),
|
||||
Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt()));
|
||||
jdbc.update("""
|
||||
INSERT INTO attachment_jobs
|
||||
(id, attachment_id, pipeline_version, attempt, status, published, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 'queued', false, ?, ?)
|
||||
""", job.jobId(), job.attachmentId(), job.pipelineVersion(), job.attempt(),
|
||||
Timestamp.from(job.createdAt()), Timestamp.from(job.createdAt()));
|
||||
return attachment;
|
||||
}
|
||||
|
||||
@@ -56,19 +68,236 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
args[i + 1] = ids.get(i);
|
||||
}
|
||||
return jdbc.query("SELECT " + COLUMNS + " FROM attachments WHERE chat_id = ? AND id IN ("
|
||||
+ placeholders + ") AND status = 'ready'",
|
||||
+ placeholders + ") AND status <> 'failed'",
|
||||
(rs, rowNum) -> map(rs), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> findAllObjectKeys() {
|
||||
return new HashSet<>(jdbc.query("SELECT object_key FROM attachments",
|
||||
Set<String> keys = new HashSet<>(jdbc.query(
|
||||
"SELECT object_key FROM attachments WHERE status = 'ready'",
|
||||
(rs, rowNum) -> rs.getString("object_key")));
|
||||
keys.addAll(jdbc.query("SELECT object_key FROM attachment_variants",
|
||||
(rs, rowNum) -> rs.getString("object_key")));
|
||||
return keys;
|
||||
}
|
||||
|
||||
private static Attachment map(java.sql.ResultSet rs) throws java.sql.SQLException {
|
||||
@Override
|
||||
public Set<String> findActiveStagingKeys() {
|
||||
return new HashSet<>(jdbc.query("""
|
||||
SELECT staging_key
|
||||
FROM attachments
|
||||
WHERE status IN ('queued', 'processing') AND staging_key IS NOT NULL
|
||||
""", (rs, rowNum) -> rs.getString("staging_key")));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markPublished(UUID jobId) {
|
||||
jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET published = true, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'queued'
|
||||
""", jobId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markPublishFaultInjected(UUID jobId) {
|
||||
return jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET publish_fault_injected = true, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND publish_fault_injected = false AND status = 'queued'
|
||||
""", jobId) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AttachmentJob> findUnpublishedJobs(int limit) {
|
||||
return jdbc.query("""
|
||||
SELECT id, attachment_id, pipeline_version, attempt, created_at
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'queued' AND published = false
|
||||
AND next_attempt_at <= clock_timestamp()
|
||||
ORDER BY next_attempt_at, created_at, id
|
||||
LIMIT ?
|
||||
""", (rs, rowNum) -> mapJob(rs), limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int currentAttempt(UUID jobId) {
|
||||
Integer attempt = jdbc.query("SELECT attempt FROM attachment_jobs WHERE id = ?",
|
||||
(rs, rowNum) -> rs.getInt("attempt"), jobId).stream().findFirst().orElse(null);
|
||||
return attempt == null ? 1 : attempt;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean claim(UUID jobId, UUID attachmentId, int attempt,
|
||||
UUID leaseToken, Duration leaseDuration) {
|
||||
int claimed = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing', attempt = GREATEST(attempt, ?),
|
||||
lease_token = ?,
|
||||
lease_until = clock_timestamp() + (? * interval '1 millisecond'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ?
|
||||
AND (status = 'queued'
|
||||
OR (status = 'processing' AND lease_until < clock_timestamp()))
|
||||
""", attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId);
|
||||
if (claimed == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status NOT IN ('ready', 'failed')
|
||||
""", attachmentId);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken,
|
||||
int nextAttempt, String errorCode, Instant nextAttemptAt) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = ?, published = false,
|
||||
last_error_code = ?, next_attempt_at = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt),
|
||||
jobId, attachmentId, leaseToken);
|
||||
if (updated == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status NOT IN ('ready', 'failed')
|
||||
""", attachmentId);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion,
|
||||
List<AttachmentVariant> variants, boolean injectCrashAfterCommit) {
|
||||
boolean triggerCrash = injectCrashAfterCommit && Boolean.TRUE.equals(jdbc.query("""
|
||||
SELECT NOT result_fault_injected
|
||||
FROM attachment_jobs
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing' AND lease_token = ?
|
||||
FOR UPDATE
|
||||
""", (rs, rowNum) -> rs.getBoolean(1), jobId, attachmentId, leaseToken)
|
||||
.stream().findFirst().orElse(false));
|
||||
int completed = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', published = true, last_error_code = NULL,
|
||||
lease_token = NULL, lease_until = NULL,
|
||||
result_fault_injected = result_fault_injected OR ?,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", triggerCrash, jobId, attachmentId, leaseToken);
|
||||
if (completed == 0) {
|
||||
return new CompletionResult(false, false);
|
||||
}
|
||||
for (AttachmentVariant variant : variants) {
|
||||
jdbc.update("""
|
||||
INSERT INTO attachment_variants
|
||||
(attachment_id, pipeline_version, name, content_type, size_bytes,
|
||||
width, height, sha256, object_key, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp())
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
object_key = EXCLUDED.object_key
|
||||
""", attachmentId, pipelineVersion, variant.name(), variant.contentType(),
|
||||
variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(),
|
||||
variant.objectKey());
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'ready', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'failed'
|
||||
""", attachmentId);
|
||||
return new CompletionResult(true, triggerCrash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', published = true, last_error_code = ?,
|
||||
lease_token = NULL, lease_until = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", errorCode, jobId, attachmentId, leaseToken);
|
||||
if (updated == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = ?, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AttachmentJob> findExpiredJobs(int limit) {
|
||||
return jdbc.query("""
|
||||
SELECT id, attachment_id, pipeline_version, attempt, created_at
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'processing' AND lease_until < clock_timestamp()
|
||||
ORDER BY lease_until, id
|
||||
LIMIT ?
|
||||
""", (rs, rowNum) -> mapJob(rs), limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt,
|
||||
String errorCode, Instant nextAttemptAt) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = ?, published = false,
|
||||
last_error_code = ?, next_attempt_at = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_until < clock_timestamp()
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId);
|
||||
if (updated == 1) {
|
||||
jdbc.update("""
|
||||
UPDATE attachments SET status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing'
|
||||
""", attachmentId);
|
||||
}
|
||||
return updated == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean failExpired(UUID jobId, UUID attachmentId, String errorCode) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', published = true, last_error_code = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_until < clock_timestamp()
|
||||
""", errorCode, jobId, attachmentId);
|
||||
if (updated == 1) {
|
||||
jdbc.update("""
|
||||
UPDATE attachments SET status = 'failed', error_code = ?,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId);
|
||||
}
|
||||
return updated == 1;
|
||||
}
|
||||
|
||||
private Attachment map(ResultSet rs) throws SQLException {
|
||||
UUID id = rs.getObject("id", UUID.class);
|
||||
return new Attachment(
|
||||
rs.getObject("id", UUID.class),
|
||||
id,
|
||||
rs.getObject("chat_id", UUID.class),
|
||||
rs.getObject("owner_id", UUID.class),
|
||||
rs.getString("filename"),
|
||||
@@ -76,7 +305,34 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
rs.getLong("size_bytes"),
|
||||
rs.getString("sha256"),
|
||||
rs.getString("object_key"),
|
||||
rs.getString("staging_key"),
|
||||
rs.getString("status"),
|
||||
findVariants(id),
|
||||
rs.getString("error_code"),
|
||||
rs.getTimestamp("created_at").toInstant(),
|
||||
rs.getTimestamp("updated_at").toInstant());
|
||||
}
|
||||
|
||||
private List<AttachmentVariant> findVariants(UUID attachmentId) {
|
||||
return jdbc.query("""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = ? AND pipeline_version = 1
|
||||
ORDER BY name
|
||||
""", (rs, rowNum) -> new AttachmentVariant(
|
||||
rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"),
|
||||
rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"),
|
||||
rs.getString("object_key")), attachmentId);
|
||||
}
|
||||
|
||||
private static AttachmentJob mapJob(ResultSet rs) throws SQLException {
|
||||
return new AttachmentJob(
|
||||
AttachmentJob.SCHEMA_VERSION,
|
||||
rs.getObject("id", UUID.class),
|
||||
AttachmentJob.TYPE,
|
||||
rs.getObject("attachment_id", UUID.class),
|
||||
rs.getInt("pipeline_version"),
|
||||
rs.getInt("attempt"),
|
||||
rs.getTimestamp("created_at").toInstant());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -16,6 +17,7 @@ import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
|
||||
public class AttachmentReconciler {
|
||||
private static final Logger log = LoggerFactory.getLogger(AttachmentReconciler.class);
|
||||
private static final String PREFIX = "attachments/";
|
||||
|
||||
@@ -2,6 +2,9 @@ package com.example.mtusi_messenger.service;
|
||||
|
||||
import com.example.mtusi_messenger.exception.ApiExceptions;
|
||||
import com.example.mtusi_messenger.model.Attachment;
|
||||
import com.example.mtusi_messenger.model.AttachmentJob;
|
||||
import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
import com.example.mtusi_messenger.queue.AttachmentJobPublisher;
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorageException;
|
||||
@@ -34,25 +37,30 @@ public class AttachmentService {
|
||||
private final AttachmentRepository attachments;
|
||||
private final ChatService chats;
|
||||
private final ObjectStorage storage;
|
||||
private final AttachmentJobPublisher publisher;
|
||||
private final TransactionTemplate transaction;
|
||||
private final long maxUploadBytes;
|
||||
private final long maxImagePixels;
|
||||
private final Semaphore uploadPermits;
|
||||
private final long uploadPermitWaitMillis;
|
||||
private final boolean injectCrashAfterOutboxCommit;
|
||||
|
||||
public AttachmentService(
|
||||
AttachmentRepository attachments,
|
||||
ChatService chats,
|
||||
ObjectStorage storage,
|
||||
AttachmentJobPublisher publisher,
|
||||
TransactionTemplate transaction,
|
||||
@Value("${messenger.attachments.max-upload-bytes:10485760}") long maxUploadBytes,
|
||||
@Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels,
|
||||
@Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads,
|
||||
@Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis
|
||||
@Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis,
|
||||
@Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit
|
||||
) {
|
||||
this.attachments = attachments;
|
||||
this.chats = chats;
|
||||
this.storage = storage;
|
||||
this.publisher = publisher;
|
||||
this.transaction = transaction;
|
||||
this.maxUploadBytes = maxUploadBytes;
|
||||
this.maxImagePixels = maxImagePixels;
|
||||
@@ -61,6 +69,7 @@ public class AttachmentService {
|
||||
}
|
||||
this.uploadPermits = new Semaphore(maxConcurrentUploads, true);
|
||||
this.uploadPermitWaitMillis = uploadPermitWaitMillis;
|
||||
this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit;
|
||||
}
|
||||
|
||||
public Attachment upload(UUID chatId,
|
||||
@@ -108,16 +117,36 @@ public class AttachmentService {
|
||||
|
||||
UUID id = UuidCreator.getTimeOrderedEpochPlus1();
|
||||
String objectKey = "attachments/" + id + "/original";
|
||||
String stagingKey = "staging/" + id + "/original";
|
||||
Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS);
|
||||
AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, now);
|
||||
Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType,
|
||||
bytes.length, sha256(bytes), objectKey, now, now);
|
||||
bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(),
|
||||
null, now, now);
|
||||
|
||||
storage.put(objectKey, bytes, contentType);
|
||||
storage.put(stagingKey, bytes, contentType);
|
||||
try {
|
||||
return transaction.execute(status -> attachments.save(attachment));
|
||||
} catch (RuntimeException databaseFailure) {
|
||||
Attachment saved = transaction.execute(status -> attachments.saveQueued(attachment, job));
|
||||
if (injectCrashAfterOutboxCommit && attachments.markPublishFaultInjected(job.jobId())) {
|
||||
// Opt-in fault injection for the exact DB COMMIT -> broker publish window.
|
||||
// The persisted outbox row remains unpublished and is recovered after restart.
|
||||
Runtime.getRuntime().halt(87);
|
||||
}
|
||||
try {
|
||||
storage.delete(objectKey);
|
||||
publisher.publish(job);
|
||||
attachments.markPublished(job.jobId());
|
||||
return saved;
|
||||
} catch (RuntimeException publishFailure) {
|
||||
// The database job and staging object are intentionally retained.
|
||||
// The scheduled republisher closes the DB-commit -> broker window.
|
||||
throw ApiExceptions.unavailable("Attachment was staged; queue publication is pending");
|
||||
}
|
||||
} catch (RuntimeException databaseFailure) {
|
||||
if (databaseFailure instanceof com.example.mtusi_messenger.exception.ApiException) {
|
||||
throw databaseFailure;
|
||||
}
|
||||
try {
|
||||
storage.delete(stagingKey);
|
||||
} catch (RuntimeException cleanupFailure) {
|
||||
databaseFailure.addSuppressed(cleanupFailure);
|
||||
}
|
||||
@@ -133,8 +162,28 @@ public class AttachmentService {
|
||||
}
|
||||
|
||||
public Download download(UUID attachmentId, UUID userId) {
|
||||
return download(attachmentId, userId, "original");
|
||||
}
|
||||
|
||||
public Download download(UUID attachmentId, UUID userId, String variantName) {
|
||||
Attachment attachment = getAccessible(attachmentId, userId);
|
||||
return new Download(attachment, storage.get(attachment.objectKey()));
|
||||
if (!"ready".equals(attachment.status())) {
|
||||
throw ApiExceptions.conflict("Attachment content is not ready");
|
||||
}
|
||||
if ("original".equals(variantName)) {
|
||||
return new Download(attachment, attachment.contentType(), attachment.filename(),
|
||||
storage.get(attachment.objectKey()));
|
||||
}
|
||||
if (!List.of("thumbnail", "crop", "optimized").contains(variantName)) {
|
||||
throw ApiExceptions.invalid("Unsupported attachment variant");
|
||||
}
|
||||
AttachmentVariant variant = attachment.variants().stream()
|
||||
.filter(candidate -> candidate.name().equals(variantName))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> ApiExceptions.notFound("Attachment variant not found"));
|
||||
String extension = "image/webp".equals(variant.contentType()) ? ".webp" : ".jpg";
|
||||
return new Download(attachment, variant.contentType(), variantName + extension,
|
||||
storage.get(variant.objectKey()));
|
||||
}
|
||||
|
||||
public void validateForMessage(UUID chatId, List<UUID> attachmentIds) {
|
||||
@@ -234,6 +283,6 @@ public class AttachmentService {
|
||||
}
|
||||
}
|
||||
|
||||
public record Download(Attachment attachment, byte[] bytes) {
|
||||
public record Download(Attachment attachment, String contentType, String filename, byte[] bytes) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package com.example.mtusi_messenger.service;
|
||||
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.Set;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
|
||||
public class StagingCleaner {
|
||||
private static final Logger log = LoggerFactory.getLogger(StagingCleaner.class);
|
||||
private static final String PREFIX = "staging/";
|
||||
private final AttachmentRepository attachments;
|
||||
private final ObjectStorage storage;
|
||||
private final Duration retention;
|
||||
private final Clock clock;
|
||||
|
||||
@Autowired
|
||||
public StagingCleaner(
|
||||
AttachmentRepository attachments,
|
||||
ObjectStorage storage,
|
||||
@Value("${messenger.attachments.staging-retention-seconds:86400}") long retentionSeconds) {
|
||||
this(attachments, storage, Duration.ofSeconds(retentionSeconds), Clock.systemUTC());
|
||||
}
|
||||
|
||||
StagingCleaner(
|
||||
AttachmentRepository attachments,
|
||||
ObjectStorage storage,
|
||||
Duration retention,
|
||||
Clock clock) {
|
||||
if (retention.isZero() || retention.isNegative()) {
|
||||
throw new IllegalArgumentException("Staging retention must be positive");
|
||||
}
|
||||
this.attachments = attachments;
|
||||
this.storage = storage;
|
||||
this.retention = retention;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
@Scheduled(
|
||||
fixedDelayString = "${messenger.attachments.reconcile-interval-millis:600000}",
|
||||
initialDelayString = "${messenger.attachments.reconcile-initial-delay-millis:60000}")
|
||||
public void cleanup() {
|
||||
try {
|
||||
Set<String> active = attachments.findActiveStagingKeys();
|
||||
var deleteBefore = clock.instant().minus(retention);
|
||||
for (ObjectStorage.StoredObject object : storage.list(PREFIX)) {
|
||||
if (!active.contains(object.objectKey()) && object.lastModified().isBefore(deleteBefore)) {
|
||||
storage.delete(object.objectKey());
|
||||
}
|
||||
}
|
||||
} catch (RuntimeException error) {
|
||||
log.warn("Staging cleanup postponed: {}", error.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package com.example.mtusi_messenger.worker;
|
||||
|
||||
import com.example.mtusi_messenger.model.Attachment;
|
||||
import com.example.mtusi_messenger.model.AttachmentJob;
|
||||
import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
import com.example.mtusi_messenger.queue.RetryPolicy;
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "messenger.role", havingValue = "worker")
|
||||
public class AttachmentWorker {
|
||||
private static final Logger log = LoggerFactory.getLogger(AttachmentWorker.class);
|
||||
private final ObjectMapper mapper;
|
||||
private final AttachmentRepository attachments;
|
||||
private final ObjectStorage storage;
|
||||
private final ImageVariantProcessor images;
|
||||
private final RetryPolicy retryPolicy;
|
||||
private final TransactionTemplate transaction;
|
||||
private final int maxAttempts;
|
||||
private final Duration jobTimeout;
|
||||
private final Duration leaseDuration;
|
||||
private final boolean injectCrashAfterResultCommit;
|
||||
private final ExecutorService processingPool;
|
||||
private final Clock clock = Clock.systemUTC();
|
||||
|
||||
public AttachmentWorker(
|
||||
ObjectMapper mapper,
|
||||
AttachmentRepository attachments,
|
||||
ObjectStorage storage,
|
||||
ImageVariantProcessor images,
|
||||
RetryPolicy retryPolicy,
|
||||
TransactionTemplate transaction,
|
||||
@Value("${messenger.jobs.max-attempts:5}") int maxAttempts,
|
||||
@Value("${messenger.jobs.timeout:60s}") Duration jobTimeout,
|
||||
@Value("${messenger.jobs.lease-duration:90s}") Duration leaseDuration,
|
||||
@Value("${messenger.jobs.fault-crash-after-result-commit:false}") boolean injectCrashAfterResultCommit,
|
||||
@Value("${spring.rabbitmq.listener.simple.concurrency:2}") int concurrency) {
|
||||
if (maxAttempts < 1 || jobTimeout.isZero() || jobTimeout.isNegative()
|
||||
|| leaseDuration.compareTo(jobTimeout) <= 0 || concurrency < 1) {
|
||||
throw new IllegalArgumentException("Invalid attachment worker limits");
|
||||
}
|
||||
this.mapper = mapper;
|
||||
this.attachments = attachments;
|
||||
this.storage = storage;
|
||||
this.images = images;
|
||||
this.retryPolicy = retryPolicy;
|
||||
this.transaction = transaction;
|
||||
this.maxAttempts = maxAttempts;
|
||||
this.jobTimeout = jobTimeout;
|
||||
this.leaseDuration = leaseDuration;
|
||||
this.injectCrashAfterResultCommit = injectCrashAfterResultCommit;
|
||||
this.processingPool = Executors.newFixedThreadPool(concurrency,
|
||||
Thread.ofPlatform().name("attachment-image-", 0).factory());
|
||||
}
|
||||
|
||||
@RabbitListener(queues = "${messenger.jobs.queue}")
|
||||
public void consume(Message message, Channel channel) throws IOException {
|
||||
long deliveryTag = message.getMessageProperties().getDeliveryTag();
|
||||
AttachmentJob job;
|
||||
try {
|
||||
job = mapper.readValue(message.getBody(), AttachmentJob.class);
|
||||
} catch (RuntimeException | IOException invalidMessage) {
|
||||
log.error("Rejecting malformed attachment job message {}",
|
||||
message.getMessageProperties().getMessageId());
|
||||
channel.basicReject(deliveryTag, false);
|
||||
return;
|
||||
}
|
||||
|
||||
Attachment attachment = attachments.findById(job.attachmentId()).orElse(null);
|
||||
if (attachment == null || "ready".equals(attachment.status()) || "failed".equals(attachment.status())) {
|
||||
channel.basicAck(deliveryTag, false);
|
||||
return;
|
||||
}
|
||||
|
||||
int attempt = Math.max(job.attempt(), attachments.currentAttempt(job.jobId()));
|
||||
UUID leaseToken = UUID.randomUUID();
|
||||
boolean claimed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.claim(job.jobId(), job.attachmentId(), attempt, leaseToken, leaseDuration)));
|
||||
if (!claimed) {
|
||||
// A live worker owns the lease, or a terminal state won the race.
|
||||
// If the owner died, the DB lease reaper will create a new outbox delivery.
|
||||
channel.basicAck(deliveryTag, false);
|
||||
return;
|
||||
}
|
||||
|
||||
Future<ProcessOutcome> future = processingPool.submit(() -> process(job, leaseToken));
|
||||
try {
|
||||
ProcessOutcome outcome = future.get(jobTimeout.toMillis(), TimeUnit.MILLISECONDS);
|
||||
if (outcome.crashAfterCommit()) {
|
||||
log.error("Fault injection: halting worker after durable result and before consumer ack");
|
||||
Runtime.getRuntime().halt(86);
|
||||
}
|
||||
channel.basicAck(deliveryTag, false);
|
||||
} catch (TimeoutException timeout) {
|
||||
future.cancel(true);
|
||||
retryOrFail(job, leaseToken, attempt, "processing_timeout", deliveryTag, channel);
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
future.cancel(true);
|
||||
channel.basicNack(deliveryTag, false, true);
|
||||
} catch (ExecutionException execution) {
|
||||
Throwable cause = execution.getCause();
|
||||
if (cause instanceof ImageVariantProcessor.ProcessingException imageFailure
|
||||
&& imageFailure.permanent()) {
|
||||
failPermanent(job, leaseToken, imageFailure.errorCode(), deliveryTag, channel);
|
||||
} else {
|
||||
String code = cause instanceof ImageVariantProcessor.ProcessingException imageFailure
|
||||
? imageFailure.errorCode() : "processing_failed";
|
||||
retryOrFail(job, leaseToken, attempt, code, deliveryTag, channel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ProcessOutcome process(AttachmentJob job, UUID leaseToken) {
|
||||
Attachment attachment = attachments.findById(job.attachmentId())
|
||||
.orElseThrow(() -> new ImageVariantProcessor.ProcessingException(
|
||||
"attachment_missing", true, "Attachment metadata is missing", null));
|
||||
byte[] original = storage.get(attachment.stagingKey());
|
||||
if (original.length != attachment.sizeBytes() || !sha256(original).equals(attachment.sha256())) {
|
||||
throw new ImageVariantProcessor.ProcessingException(
|
||||
"invalid_image", true, "Staging checksum mismatch", null);
|
||||
}
|
||||
|
||||
storage.put(attachment.objectKey(), original, attachment.contentType());
|
||||
List<ImageVariantProcessor.GeneratedVariant> generated = images.createVariants(original);
|
||||
List<AttachmentVariant> variants = new ArrayList<>(generated.size());
|
||||
for (ImageVariantProcessor.GeneratedVariant item : generated) {
|
||||
String objectKey = "attachments/" + attachment.id() + "/"
|
||||
+ item.name() + "-v" + job.pipelineVersion() + "." + item.extension();
|
||||
storage.put(objectKey, item.bytes(), item.contentType());
|
||||
variants.add(new AttachmentVariant(
|
||||
item.name(), item.contentType(), item.bytes().length,
|
||||
item.width(), item.height(), sha256(item.bytes()), objectKey));
|
||||
}
|
||||
|
||||
AttachmentRepository.CompletionResult completion = transaction.execute(status ->
|
||||
attachments.markReady(job.jobId(), job.attachmentId(), leaseToken,
|
||||
job.pipelineVersion(), variants, injectCrashAfterResultCommit));
|
||||
if (completion == null || !completion.completed()) {
|
||||
return new ProcessOutcome(false);
|
||||
}
|
||||
try {
|
||||
storage.delete(attachment.stagingKey());
|
||||
} catch (RuntimeException cleanupFailure) {
|
||||
log.warn("Staging cleanup postponed for attachment {}", attachment.id());
|
||||
}
|
||||
return new ProcessOutcome(completion.crashAfterCommit());
|
||||
}
|
||||
|
||||
private void failPermanent(
|
||||
AttachmentJob job, UUID leaseToken, String errorCode,
|
||||
long deliveryTag, Channel channel) throws IOException {
|
||||
boolean failed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, errorCode)));
|
||||
if (failed) {
|
||||
channel.basicReject(deliveryTag, false);
|
||||
} else {
|
||||
channel.basicAck(deliveryTag, false);
|
||||
}
|
||||
}
|
||||
|
||||
private void retryOrFail(
|
||||
AttachmentJob job, UUID leaseToken, int attempt, String errorCode,
|
||||
long deliveryTag, Channel channel) throws IOException {
|
||||
try {
|
||||
if (attempt >= maxAttempts) {
|
||||
boolean failed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken,
|
||||
"retry_exhausted")));
|
||||
if (failed) {
|
||||
log.error("Attachment job {} exhausted {} attempts: {}",
|
||||
job.jobId(), attempt, errorCode);
|
||||
channel.basicReject(deliveryTag, false);
|
||||
} else {
|
||||
channel.basicAck(deliveryTag, false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
int nextAttempt = attempt + 1;
|
||||
boolean queued = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markQueuedForRetry(
|
||||
job.jobId(), job.attachmentId(), leaseToken, nextAttempt, errorCode,
|
||||
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)))));
|
||||
// The DB outbox is now the durable retry carrier; the current broker
|
||||
// delivery can be acknowledged without a DB/broker dual write.
|
||||
channel.basicAck(deliveryTag, false);
|
||||
if (!queued) {
|
||||
log.info("Retry skipped because job {} lost its lease or became terminal", job.jobId());
|
||||
}
|
||||
} catch (RuntimeException databaseFailure) {
|
||||
channel.basicNack(deliveryTag, false, true);
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void shutdownPool() {
|
||||
processingPool.shutdownNow();
|
||||
}
|
||||
|
||||
private static String sha256(byte[] bytes) {
|
||||
try {
|
||||
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
|
||||
} catch (NoSuchAlgorithmException impossible) {
|
||||
throw new IllegalStateException("SHA-256 is unavailable", impossible);
|
||||
}
|
||||
}
|
||||
|
||||
private record ProcessOutcome(boolean crashAfterCommit) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package com.example.mtusi_messenger.worker;
|
||||
|
||||
import net.coobird.thumbnailator.Thumbnails;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.imageio.IIOImage;
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.ImageWriteParam;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Component
|
||||
public class ImageVariantProcessor {
|
||||
private static final long MEBIBYTE = 1024L * 1024L;
|
||||
private static final long OUTPUT_ALLOWANCE_BYTES = 64L * MEBIBYTE;
|
||||
private final long maxPixels;
|
||||
private final long memoryBudgetBytes;
|
||||
private final Semaphore memoryPermits;
|
||||
private final Duration memoryWait;
|
||||
|
||||
public ImageVariantProcessor(
|
||||
@Value("${messenger.jobs.worker-max-pixels:40000000}") long maxPixels,
|
||||
@Value("${messenger.jobs.worker-memory-budget-bytes:536870912}") long memoryBudgetBytes,
|
||||
@Value("${messenger.jobs.memory-wait:2s}") Duration memoryWait) {
|
||||
if (maxPixels < 1 || memoryBudgetBytes < MEBIBYTE || memoryWait.isNegative()) {
|
||||
throw new IllegalArgumentException("Invalid worker image resource limits");
|
||||
}
|
||||
this.maxPixels = maxPixels;
|
||||
this.memoryBudgetBytes = memoryBudgetBytes;
|
||||
this.memoryPermits = new Semaphore((int) Math.min(Integer.MAX_VALUE,
|
||||
Math.max(1, memoryBudgetBytes / MEBIBYTE)), true);
|
||||
this.memoryWait = memoryWait;
|
||||
}
|
||||
|
||||
public List<GeneratedVariant> createVariants(byte[] original) {
|
||||
ImageInfo info = inspect(original);
|
||||
long estimatedBytes;
|
||||
try {
|
||||
estimatedBytes = Math.addExact(Math.multiplyExact(info.pixels(), 12L),
|
||||
OUTPUT_ALLOWANCE_BYTES);
|
||||
} catch (ArithmeticException overflow) {
|
||||
throw permanent("image_too_large", "Image memory estimate overflow", overflow);
|
||||
}
|
||||
if (estimatedBytes > memoryBudgetBytes) {
|
||||
throw permanent("image_too_large", "Image exceeds worker memory budget", null);
|
||||
}
|
||||
int permits = (int) Math.max(1, (estimatedBytes + MEBIBYTE - 1) / MEBIBYTE);
|
||||
boolean acquired;
|
||||
try {
|
||||
acquired = memoryPermits.tryAcquire(permits, memoryWait.toMillis(), TimeUnit.MILLISECONDS);
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw temporary("worker_interrupted", "Interrupted while waiting for memory budget", interrupted);
|
||||
}
|
||||
if (!acquired) {
|
||||
throw temporary("worker_capacity", "Worker memory budget is currently exhausted", null);
|
||||
}
|
||||
|
||||
try {
|
||||
return transform(original);
|
||||
} finally {
|
||||
memoryPermits.release(permits);
|
||||
}
|
||||
}
|
||||
|
||||
private ImageInfo inspect(byte[] bytes) {
|
||||
ImageIO.setUseCache(false);
|
||||
try (var input = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) {
|
||||
if (input == null) {
|
||||
throw permanent("invalid_image", "Image input is invalid", null);
|
||||
}
|
||||
Iterator<ImageReader> readers = ImageIO.getImageReaders(input);
|
||||
if (!readers.hasNext()) {
|
||||
throw permanent("invalid_image", "Unsupported image encoding", null);
|
||||
}
|
||||
ImageReader reader = readers.next();
|
||||
try {
|
||||
reader.setInput(input, true, true);
|
||||
int width = reader.getWidth(0);
|
||||
int height = reader.getHeight(0);
|
||||
long pixels = Math.multiplyExact((long) width, height);
|
||||
if (width < 1 || height < 1 || pixels > maxPixels) {
|
||||
throw permanent("image_too_large", "Image exceeds worker pixel limit", null);
|
||||
}
|
||||
return new ImageInfo(width, height, pixels);
|
||||
} finally {
|
||||
reader.dispose();
|
||||
}
|
||||
} catch (ProcessingException error) {
|
||||
throw error;
|
||||
} catch (IOException | RuntimeException error) {
|
||||
throw permanent("invalid_image", "Image header cannot be read", error);
|
||||
}
|
||||
}
|
||||
|
||||
private List<GeneratedVariant> transform(byte[] original) {
|
||||
try {
|
||||
BufferedImage oriented = Thumbnails.of(new ByteArrayInputStream(original))
|
||||
.useExifOrientation(true)
|
||||
.scale(1.0)
|
||||
.asBufferedImage();
|
||||
if (oriented == null || oriented.getWidth() < 1 || oriented.getHeight() < 1) {
|
||||
throw permanent("invalid_image", "Image cannot be decoded", null);
|
||||
}
|
||||
try {
|
||||
List<GeneratedVariant> variants = new ArrayList<>(3);
|
||||
BufferedImage thumbnail = fit(oriented, 256, BufferedImage.TYPE_INT_RGB, true);
|
||||
variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", 0.85f));
|
||||
thumbnail.flush();
|
||||
|
||||
BufferedImage crop = cropSquare(oriented, 128);
|
||||
variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", 0.88f));
|
||||
crop.flush();
|
||||
|
||||
int webpType = oriented.getColorModel().hasAlpha()
|
||||
? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB;
|
||||
BufferedImage optimized = fit(oriented, 1280, webpType, !oriented.getColorModel().hasAlpha());
|
||||
variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", 0.82f));
|
||||
optimized.flush();
|
||||
return List.copyOf(variants);
|
||||
} finally {
|
||||
oriented.flush();
|
||||
}
|
||||
} catch (ProcessingException error) {
|
||||
throw error;
|
||||
} catch (IOException | RuntimeException error) {
|
||||
throw permanent("invalid_image", "Image cannot be transformed", error);
|
||||
}
|
||||
}
|
||||
|
||||
private static BufferedImage fit(BufferedImage source, int maxSide, int type, boolean whiteBackground) {
|
||||
double scale = Math.min(1.0, Math.min(
|
||||
(double) maxSide / source.getWidth(),
|
||||
(double) maxSide / source.getHeight()));
|
||||
int width = Math.max(1, (int) Math.floor(source.getWidth() * scale));
|
||||
int height = Math.max(1, (int) Math.floor(source.getHeight() * scale));
|
||||
BufferedImage target = new BufferedImage(width, height, type);
|
||||
Graphics2D graphics = qualityGraphics(target, whiteBackground);
|
||||
try {
|
||||
graphics.drawImage(source, 0, 0, width, height, null);
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
private static BufferedImage cropSquare(BufferedImage source, int side) {
|
||||
int sourceSide = Math.min(source.getWidth(), source.getHeight());
|
||||
int x = (source.getWidth() - sourceSide) / 2;
|
||||
int y = (source.getHeight() - sourceSide) / 2;
|
||||
BufferedImage target = new BufferedImage(side, side, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D graphics = qualityGraphics(target, true);
|
||||
try {
|
||||
graphics.drawImage(source, 0, 0, side, side,
|
||||
x, y, x + sourceSide, y + sourceSide, null);
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
private static Graphics2D qualityGraphics(BufferedImage target, boolean whiteBackground) {
|
||||
Graphics2D graphics = target.createGraphics();
|
||||
if (whiteBackground) {
|
||||
graphics.setColor(Color.WHITE);
|
||||
graphics.fillRect(0, 0, target.getWidth(), target.getHeight());
|
||||
}
|
||||
graphics.setRenderingHint(RenderingHints.KEY_INTERPOLATION,
|
||||
RenderingHints.VALUE_INTERPOLATION_BICUBIC);
|
||||
graphics.setRenderingHint(RenderingHints.KEY_RENDERING,
|
||||
RenderingHints.VALUE_RENDER_QUALITY);
|
||||
graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
|
||||
RenderingHints.VALUE_ANTIALIAS_ON);
|
||||
return graphics;
|
||||
}
|
||||
|
||||
private static GeneratedVariant encode(
|
||||
String name, String contentType, String extension,
|
||||
BufferedImage image, String format, float quality) throws IOException {
|
||||
Iterator<javax.imageio.ImageWriter> writers = ImageIO.getImageWritersByFormatName(format);
|
||||
if (!writers.hasNext()) {
|
||||
throw new IOException(format + " writer is unavailable");
|
||||
}
|
||||
var writer = writers.next();
|
||||
try (var output = new ByteArrayOutputStream();
|
||||
var imageOutput = ImageIO.createImageOutputStream(output)) {
|
||||
writer.setOutput(imageOutput);
|
||||
ImageWriteParam parameters = writer.getDefaultWriteParam();
|
||||
if (parameters.canWriteCompressed()) {
|
||||
parameters.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
|
||||
String[] compressionTypes = parameters.getCompressionTypes();
|
||||
if (compressionTypes != null && compressionTypes.length > 0) {
|
||||
parameters.setCompressionType(compressionTypes[0]);
|
||||
}
|
||||
parameters.setCompressionQuality(quality);
|
||||
}
|
||||
// Source metadata is intentionally omitted for every generated variant.
|
||||
writer.write(null, new IIOImage(image, null, null), parameters);
|
||||
imageOutput.flush();
|
||||
return new GeneratedVariant(name, contentType, extension, output.toByteArray(),
|
||||
image.getWidth(), image.getHeight());
|
||||
} finally {
|
||||
writer.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static ProcessingException permanent(String code, String message, Throwable cause) {
|
||||
return new ProcessingException(code, true, message, cause);
|
||||
}
|
||||
|
||||
private static ProcessingException temporary(String code, String message, Throwable cause) {
|
||||
return new ProcessingException(code, false, message, cause);
|
||||
}
|
||||
|
||||
private record ImageInfo(int width, int height, long pixels) {
|
||||
}
|
||||
|
||||
public record GeneratedVariant(String name, String contentType, String extension,
|
||||
byte[] bytes, int width, int height) {
|
||||
}
|
||||
|
||||
public static final class ProcessingException extends RuntimeException {
|
||||
private final String errorCode;
|
||||
private final boolean permanent;
|
||||
|
||||
public ProcessingException(
|
||||
String errorCode, boolean permanent, String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
this.errorCode = errorCode;
|
||||
this.permanent = permanent;
|
||||
}
|
||||
|
||||
public String errorCode() {
|
||||
return errorCode;
|
||||
}
|
||||
|
||||
public boolean permanent() {
|
||||
return permanent;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,23 @@ spring:
|
||||
min-idle: ${REDIS_POOL_MIN_IDLE:0}
|
||||
max-wait: ${REDIS_POOL_MAX_WAIT:500ms}
|
||||
|
||||
rabbitmq:
|
||||
host: ${RABBITMQ_HOST:localhost}
|
||||
port: ${RABBITMQ_PORT:5672}
|
||||
username: ${RABBITMQ_USER:messenger}
|
||||
password: ${RABBITMQ_PASSWORD:messenger}
|
||||
virtual-host: ${RABBITMQ_VHOST:/}
|
||||
connection-timeout: ${RABBITMQ_CONNECTION_TIMEOUT:5s}
|
||||
publisher-confirm-type: correlated
|
||||
publisher-returns: true
|
||||
listener:
|
||||
simple:
|
||||
acknowledge-mode: manual
|
||||
default-requeue-rejected: false
|
||||
prefetch: ${WORKER_PREFETCH:1}
|
||||
concurrency: ${WORKER_CONCURRENCY:2}
|
||||
max-concurrency: ${WORKER_CONCURRENCY:2}
|
||||
|
||||
jdbc:
|
||||
template:
|
||||
query-timeout: ${DATABASE_QUERY_TIMEOUT_SECONDS:5}
|
||||
@@ -53,6 +70,7 @@ server:
|
||||
shutdown: graceful
|
||||
|
||||
messenger:
|
||||
role: ${MESSENGER_ROLE:api}
|
||||
storage: postgres
|
||||
session:
|
||||
ttl-seconds: ${SESSION_TTL_SECONDS:3600}
|
||||
@@ -71,7 +89,28 @@ messenger:
|
||||
max-image-pixels: ${MAX_IMAGE_PIXELS:40000000}
|
||||
max-concurrent-uploads: ${MAX_CONCURRENT_UPLOADS:3}
|
||||
upload-permit-wait-millis: ${UPLOAD_PERMIT_WAIT_MILLIS:10000}
|
||||
staging-retention-seconds: ${STAGING_RETENTION_SECONDS:86400}
|
||||
cleanup-grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600}
|
||||
cleanup-dry-run: ${ATTACHMENT_CLEANUP_DRY_RUN:false}
|
||||
reconcile-interval-millis: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:600000}
|
||||
reconcile-initial-delay-millis: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:60000}
|
||||
jobs:
|
||||
exchange: ${ATTACHMENT_JOB_EXCHANGE:attachments.v1}
|
||||
queue: ${ATTACHMENT_JOB_QUEUE:attachments.process.level4.v1}
|
||||
routing-key: ${ATTACHMENT_JOB_ROUTING_KEY:attachment.process.v1}
|
||||
dead-letter-exchange: ${ATTACHMENT_DLQ_EXCHANGE:attachments.dlx.v1}
|
||||
dead-letter-queue: ${ATTACHMENT_DLQ_QUEUE:attachments.failed.v1}
|
||||
dead-letter-routing-key: ${ATTACHMENT_DLQ_ROUTING_KEY:attachment.failed.v1}
|
||||
max-attempts: ${JOB_MAX_ATTEMPTS:5}
|
||||
retry-base-delay: ${JOB_RETRY_BASE_DELAY:1s}
|
||||
retry-max-delay: ${JOB_RETRY_MAX_DELAY:30s}
|
||||
timeout: ${JOB_TIMEOUT:60s}
|
||||
lease-duration: ${JOB_LEASE_DURATION:90s}
|
||||
lease-reaper-interval: ${JOB_LEASE_REAPER_INTERVAL:5s}
|
||||
worker-max-pixels: ${WORKER_MAX_IMAGE_PIXELS:40000000}
|
||||
worker-memory-budget-bytes: ${WORKER_MEMORY_BUDGET_BYTES:536870912}
|
||||
memory-wait: ${WORKER_MEMORY_WAIT:2s}
|
||||
publish-confirm-timeout: ${JOB_PUBLISH_CONFIRM_TIMEOUT:5s}
|
||||
republish-interval: ${JOB_REPUBLISH_INTERVAL:5s}
|
||||
fault-crash-after-outbox-commit: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:false}
|
||||
fault-crash-after-result-commit: ${FAULT_CRASH_AFTER_RESULT_COMMIT:false}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk;
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key text,
|
||||
ADD COLUMN error_code varchar(64),
|
||||
ADD CONSTRAINT attachments_status_chk
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD CONSTRAINT attachments_error_state_chk
|
||||
CHECK ((status = 'failed' AND error_code IS NOT NULL)
|
||||
OR (status <> 'failed' AND error_code IS NULL));
|
||||
|
||||
CREATE UNIQUE INDEX attachments_staging_key_uk
|
||||
ON attachments (staging_key) WHERE staging_key IS NOT NULL;
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
name varchar(32) NOT NULL,
|
||||
content_type varchar(32) NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 char(64) NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
|
||||
CONSTRAINT attachment_variants_pk
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_object_key_uk UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_name_chk
|
||||
CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_content_type_chk
|
||||
CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
CONSTRAINT attachment_variants_size_chk CHECK (size_bytes > 0),
|
||||
CONSTRAINT attachment_variants_dimensions_chk CHECK (width > 0 AND height > 0),
|
||||
CONSTRAINT attachment_variants_sha256_chk CHECK (sha256 ~ '^[a-f0-9]{64}$')
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL DEFAULT 1,
|
||||
attempt integer NOT NULL DEFAULT 1,
|
||||
status varchar(16) NOT NULL DEFAULT 'queued',
|
||||
published boolean NOT NULL DEFAULT false,
|
||||
last_error_code varchar(64),
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
|
||||
CONSTRAINT attachment_jobs_pipeline_chk CHECK (pipeline_version > 0),
|
||||
CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt > 0),
|
||||
CONSTRAINT attachment_jobs_status_chk
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed'))
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_publish_idx
|
||||
ON attachment_jobs (published, status, created_at)
|
||||
WHERE published = false AND status = 'queued';
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN next_attempt_at timestamptz,
|
||||
ADD COLUMN lease_token uuid,
|
||||
ADD COLUMN lease_until timestamptz,
|
||||
ADD COLUMN publish_fault_injected boolean NOT NULL DEFAULT false,
|
||||
ADD COLUMN result_fault_injected boolean NOT NULL DEFAULT false;
|
||||
|
||||
UPDATE attachment_jobs SET next_attempt_at = created_at WHERE next_attempt_at IS NULL;
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', published = false, last_error_code = 'lease_migration_recovery'
|
||||
WHERE status = 'processing';
|
||||
UPDATE attachments SET status = 'queued', error_code = NULL WHERE status = 'processing';
|
||||
ALTER TABLE attachment_jobs ALTER COLUMN next_attempt_at SET NOT NULL;
|
||||
ALTER TABLE attachment_jobs ALTER COLUMN next_attempt_at SET DEFAULT clock_timestamp();
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD CONSTRAINT attachment_jobs_lease_chk CHECK (
|
||||
(status = 'processing' AND lease_token IS NOT NULL AND lease_until IS NOT NULL)
|
||||
OR (status <> 'processing' AND lease_token IS NULL AND lease_until IS NULL)
|
||||
);
|
||||
|
||||
DROP INDEX attachment_jobs_publish_idx;
|
||||
CREATE INDEX attachment_jobs_publish_idx
|
||||
ON attachment_jobs (next_attempt_at, created_at, id)
|
||||
WHERE published = false AND status = 'queued';
|
||||
CREATE INDEX attachment_jobs_expired_lease_idx
|
||||
ON attachment_jobs (lease_until)
|
||||
WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE attachment_job_replays (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE RESTRICT,
|
||||
requested_by varchar(128) NOT NULL,
|
||||
previous_attempt integer NOT NULL,
|
||||
requested_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_job_replays_actor_chk
|
||||
CHECK (char_length(requested_by) BETWEEN 1 AND 128)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_job_replays_job_idx
|
||||
ON attachment_job_replays (job_id, requested_at);
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.example.mtusi_messenger.queue;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class RetryPolicyTest {
|
||||
@Test
|
||||
void appliesExponentialBackoffWithBoundedJitter() {
|
||||
RetryPolicy policy = new RetryPolicy(Duration.ofSeconds(1), Duration.ofSeconds(30));
|
||||
|
||||
for (int attempt = 2; attempt <= 8; attempt++) {
|
||||
long ceiling = Math.min(30_000L, 1_000L << (attempt - 2));
|
||||
long delay = policy.delayForAttempt(attempt).toMillis();
|
||||
assertTrue(delay >= Math.max(1, ceiling / 2));
|
||||
assertTrue(delay <= ceiling);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import com.example.mtusi_messenger.exception.ApiException;
|
||||
import com.example.mtusi_messenger.model.Attachment;
|
||||
import com.example.mtusi_messenger.model.Chat;
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.queue.AttachmentJobPublisher;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorageException;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -34,6 +35,7 @@ class AttachmentServiceTest {
|
||||
private AttachmentRepository repository;
|
||||
private ChatService chats;
|
||||
private InMemoryStorage storage;
|
||||
private AttachmentJobPublisher publisher;
|
||||
private AttachmentService service;
|
||||
private byte[] png;
|
||||
|
||||
@@ -42,6 +44,7 @@ class AttachmentServiceTest {
|
||||
repository = mock(AttachmentRepository.class);
|
||||
chats = mock(ChatService.class);
|
||||
storage = new InMemoryStorage();
|
||||
publisher = mock(AttachmentJobPublisher.class);
|
||||
TransactionTemplate transaction = mock(TransactionTemplate.class);
|
||||
when(transaction.execute(any())).thenAnswer(invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
@@ -50,14 +53,14 @@ class AttachmentServiceTest {
|
||||
});
|
||||
when(chats.getAccessible(chatId, ownerId)).thenReturn(
|
||||
new Chat(chatId, "test", List.of(ownerId), Instant.now()));
|
||||
when(repository.save(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
service = new AttachmentService(repository, chats, storage, transaction,
|
||||
10_485_760, 40_000_000, 3, 10_000);
|
||||
when(repository.saveQueued(any(), any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
service = new AttachmentService(repository, chats, storage, publisher, transaction,
|
||||
10_485_760, 40_000_000, 3, 10_000, false);
|
||||
png = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void uploadsValidPngUsingServerGeneratedObjectKey() {
|
||||
void stagesValidPngAndPublishesDurableJob() {
|
||||
Attachment attachment = service.upload(chatId, ownerId, "photo.png", "image/png",
|
||||
png.length, new ByteArrayInputStream(png));
|
||||
|
||||
@@ -65,9 +68,13 @@ class AttachmentServiceTest {
|
||||
assertEquals("image/png", attachment.contentType());
|
||||
assertEquals(png.length, attachment.sizeBytes());
|
||||
assertTrue(attachment.objectKey().matches("attachments/[0-9a-f-]+/original"));
|
||||
assertTrue(attachment.stagingKey().matches("staging/[0-9a-f-]+/original"));
|
||||
assertEquals("queued", attachment.status());
|
||||
assertFalse(attachment.objectKey().contains("photo.png"));
|
||||
assertArrayEquals(png, storage.objects.get(attachment.objectKey()));
|
||||
verify(repository).save(attachment);
|
||||
assertArrayEquals(png, storage.objects.get(attachment.stagingKey()));
|
||||
assertFalse(storage.objects.containsKey(attachment.objectKey()));
|
||||
verify(repository).saveQueued(eq(attachment), any());
|
||||
verify(publisher).publish(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -108,7 +115,7 @@ class AttachmentServiceTest {
|
||||
assertThrows(ObjectStorageException.class, () -> service.upload(
|
||||
chatId, ownerId, "photo.png", "image/png", png.length,
|
||||
new ByteArrayInputStream(png)));
|
||||
verify(repository, never()).save(any());
|
||||
verify(repository, never()).saveQueued(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -130,12 +137,12 @@ class AttachmentServiceTest {
|
||||
|
||||
assertEquals(HttpStatus.BAD_REQUEST, error.status());
|
||||
assertTrue(storage.objects.isEmpty());
|
||||
verify(repository, never()).save(any());
|
||||
verify(repository, never()).saveQueued(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void databaseFailureCompensatesAlreadyWrittenS3Object() {
|
||||
when(repository.save(any())).thenThrow(new IllegalStateException("database commit failed"));
|
||||
when(repository.saveQueued(any(), any())).thenThrow(new IllegalStateException("database commit failed"));
|
||||
|
||||
assertThrows(IllegalStateException.class, () -> service.upload(
|
||||
chatId, ownerId, "photo.png", "image/png", png.length,
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.example.mtusi_messenger.service;
|
||||
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class StagingCleanerTest {
|
||||
private static final Instant NOW = Instant.parse("2026-11-18T12:00:00Z");
|
||||
|
||||
@Test
|
||||
void keepsActiveAndFreshStagingButDeletesExpiredUnreferencedObject() {
|
||||
AttachmentRepository repository = mock(AttachmentRepository.class);
|
||||
when(repository.findActiveStagingKeys()).thenReturn(Set.of("staging/active/original"));
|
||||
FakeStorage storage = new FakeStorage(List.of(
|
||||
object("staging/active/original", NOW.minusSeconds(172800)),
|
||||
object("staging/failed/original", NOW.minusSeconds(172800)),
|
||||
object("staging/fresh/original", NOW.minusSeconds(60))));
|
||||
StagingCleaner cleaner = new StagingCleaner(repository, storage, Duration.ofDays(1),
|
||||
Clock.fixed(NOW, ZoneOffset.UTC));
|
||||
|
||||
cleaner.cleanup();
|
||||
|
||||
assertEquals(List.of("staging/failed/original"), storage.deleted);
|
||||
}
|
||||
|
||||
private static ObjectStorage.StoredObject object(String key, Instant modified) {
|
||||
return new ObjectStorage.StoredObject(key, modified, 100);
|
||||
}
|
||||
|
||||
private static final class FakeStorage implements ObjectStorage {
|
||||
private final List<StoredObject> objects;
|
||||
private final List<String> deleted = new ArrayList<>();
|
||||
|
||||
private FakeStorage(List<StoredObject> objects) {
|
||||
this.objects = objects;
|
||||
}
|
||||
|
||||
@Override public void put(String objectKey, byte[] bytes, String contentType) { }
|
||||
@Override public byte[] get(String objectKey) { return new byte[0]; }
|
||||
@Override public void delete(String objectKey) { deleted.add(objectKey); }
|
||||
@Override public List<StoredObject> list(String prefix) { return objects; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.example.mtusi_messenger.worker;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ImageVariantProcessorTest {
|
||||
private final byte[] original;
|
||||
|
||||
ImageVariantProcessorTest() throws Exception {
|
||||
original = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsThumbnailCropAndOptimizedWebp() throws Exception {
|
||||
ImageVariantProcessor processor = new ImageVariantProcessor(
|
||||
40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1));
|
||||
|
||||
Map<String, ImageVariantProcessor.GeneratedVariant> variants = processor
|
||||
.createVariants(original).stream()
|
||||
.collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name,
|
||||
Function.identity()));
|
||||
|
||||
assertEquals(3, variants.size());
|
||||
assertImage(variants.get("thumbnail"), "image/jpeg", 256, 160);
|
||||
assertImage(variants.get("crop"), "image/jpeg", 128, 128);
|
||||
assertImage(variants.get("optimized"), "image/webp", 320, 200);
|
||||
byte[] webp = variants.get("optimized").bytes();
|
||||
assertEquals("RIFF", new String(webp, 0, 4, StandardCharsets.US_ASCII));
|
||||
assertEquals("WEBP", new String(webp, 8, 4, StandardCharsets.US_ASCII));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsImageWhoseEstimateExceedsWorkerMemoryBudget() {
|
||||
ImageVariantProcessor processor = new ImageVariantProcessor(
|
||||
40_000_000, 1L * 1024 * 1024, Duration.ofSeconds(1));
|
||||
|
||||
var error = assertThrows(ImageVariantProcessor.ProcessingException.class,
|
||||
() -> processor.createVariants(original));
|
||||
|
||||
assertTrue(error.permanent());
|
||||
assertEquals("image_too_large", error.errorCode());
|
||||
}
|
||||
|
||||
private static void assertImage(
|
||||
ImageVariantProcessor.GeneratedVariant variant,
|
||||
String contentType, int width, int height) throws Exception {
|
||||
assertNotNull(variant);
|
||||
assertEquals(contentType, variant.contentType());
|
||||
assertEquals(width, variant.width());
|
||||
assertEquals(height, variant.height());
|
||||
var decoded = ImageIO.read(new ByteArrayInputStream(variant.bytes()));
|
||||
assertNotNull(decoded);
|
||||
assertEquals(width, decoded.getWidth());
|
||||
assertEquals(height, decoded.getHeight());
|
||||
}
|
||||
}
|
||||
@@ -16,3 +16,9 @@ The course-provided black-box suite remains in `tests/`; Spring tests remain in
|
||||
unknown attachment, foreign user and cross-chat binding; every user error is 4xx.
|
||||
- `concurrent_upload_memory.py` — 12 simultaneous ~6.5 MiB valid PNG uploads and
|
||||
aggregate API container memory sampling through `docker stats`.
|
||||
- `lab5_worker_recovery.py` — durable worker stop/restart, all three variants and
|
||||
idempotent duplicate delivery.
|
||||
- `lab5_fault_injection.py` — crashes after outbox commit and after result commit
|
||||
but before consumer acknowledgement.
|
||||
- `lab5_dlq_replay_lease.py` — permanent failure DLQ routing, guarded audited
|
||||
replay, and expired processing-lease recovery.
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise DLQ routing, guarded replay and expired processing leases."""
|
||||
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env.example"]
|
||||
|
||||
|
||||
def command(*args, capture=False):
|
||||
result = subprocess.run(
|
||||
[*COMPOSE, *args], cwd=ROOT, check=True, text=True,
|
||||
capture_output=capture)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def database_scalar(sql):
|
||||
return command(
|
||||
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", sql, capture=True)
|
||||
|
||||
|
||||
def env_value(name):
|
||||
for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith(name + "="):
|
||||
return line.split("=", 1)[1]
|
||||
raise Failure("missing " + name + " in .env.example")
|
||||
|
||||
|
||||
def queue_ready(queue):
|
||||
listing = command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues",
|
||||
"name", "messages_ready", capture=True)
|
||||
for line in listing.splitlines():
|
||||
columns = line.split("\t")
|
||||
if len(columns) == 2 and columns[0] == queue:
|
||||
return int(columns[1])
|
||||
raise Failure("queue not found: " + queue)
|
||||
|
||||
|
||||
def wait_ready(client, actor, attachment, seconds=120):
|
||||
return client.wait_ready(actor, attachment, wait_seconds=seconds)
|
||||
|
||||
|
||||
def replay(job_id):
|
||||
subprocess.run(
|
||||
["powershell", "-NoProfile", "-File", str(ROOT / "scripts" / "replay-failed.ps1"),
|
||||
"-JobId", job_id, "-RequestedBy", "integration-test",
|
||||
"-EnvFile", ".env.example"],
|
||||
cwd=ROOT, check=True, text=True)
|
||||
|
||||
|
||||
def main():
|
||||
client = Client("http://localhost:8080")
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
|
||||
# A permanently malformed delivery is rejected and dead-lettered.
|
||||
before_dlq = queue_ready(env_value("ATTACHMENT_DLQ_QUEUE"))
|
||||
command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqadmin",
|
||||
"-u", env_value("RABBITMQ_USER"), "-p", env_value("RABBITMQ_PASSWORD"),
|
||||
"publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"),
|
||||
"routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"),
|
||||
"payload={}",
|
||||
'properties={"delivery_mode":2,"content_type":"application/json"}')
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline and queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) <= before_dlq:
|
||||
time.sleep(0.5)
|
||||
check(queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) == before_dlq + 1,
|
||||
"malformed permanent failure was not dead-lettered exactly once")
|
||||
|
||||
# Replay is allowed only from failed and creates one audit record.
|
||||
command("stop", "worker")
|
||||
failed, _ = client.attachment(alice, chat)
|
||||
job_id = database_scalar(
|
||||
"SELECT id FROM attachment_jobs WHERE attachment_id = '" + failed["id"] + "';")
|
||||
database_scalar(
|
||||
"UPDATE attachment_jobs SET status='failed', published=true, "
|
||||
"last_error_code='retry_exhausted', lease_token=NULL, lease_until=NULL "
|
||||
"WHERE id='" + job_id + "'; "
|
||||
"UPDATE attachments SET status='failed', error_code='retry_exhausted' "
|
||||
"WHERE id='" + failed["id"] + "'; SELECT 1;")
|
||||
replay(job_id)
|
||||
replay(job_id)
|
||||
audit_count = database_scalar(
|
||||
"SELECT count(*) FROM attachment_job_replays WHERE job_id='" + job_id + "';")
|
||||
check(audit_count == "1", "replay was not guarded or was audited more than once")
|
||||
command("start", "worker")
|
||||
ready = wait_ready(client, bob, failed)
|
||||
check(ready["status"] == "ready", "replayed failed job did not become ready")
|
||||
replay(job_id)
|
||||
check(database_scalar(
|
||||
"SELECT count(*) FROM attachment_job_replays WHERE job_id='" + job_id + "';") == "1",
|
||||
"replay changed an already successful job")
|
||||
check(database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" + failed["id"] + "';") == "3",
|
||||
"replay did not preserve exactly one variant set")
|
||||
|
||||
# The API reaper returns an expired processing lease to the durable outbox.
|
||||
command("stop", "worker")
|
||||
leased, _ = client.attachment(alice, chat)
|
||||
lease_job = database_scalar(
|
||||
"SELECT id FROM attachment_jobs WHERE attachment_id='" + leased["id"] + "';")
|
||||
database_scalar(
|
||||
"UPDATE attachment_jobs SET status='processing', attempt=1, published=true, "
|
||||
"lease_token=gen_random_uuid(), lease_until=clock_timestamp()-interval '1 second' "
|
||||
"WHERE id='" + lease_job + "'; "
|
||||
"UPDATE attachments SET status='processing' WHERE id='" + leased["id"] + "'; SELECT 1;")
|
||||
command("start", "worker")
|
||||
lease_ready = wait_ready(client, bob, leased)
|
||||
check(lease_ready["status"] == "ready", "expired processing lease was not recovered")
|
||||
check(int(database_scalar(
|
||||
"SELECT attempt FROM attachment_jobs WHERE id='" + lease_job + "';")) >= 2,
|
||||
"expired lease did not consume a bounded retry attempt")
|
||||
print("PASS: DLQ, guarded replay and expired lease recovery")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove recovery across DB-commit/publish and result-commit/ack crashes."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env.example"]
|
||||
|
||||
|
||||
def command(*args, capture=False, environment=None):
|
||||
result = subprocess.run(
|
||||
[*COMPOSE, *args], cwd=ROOT, check=True, text=True,
|
||||
capture_output=capture, env=environment)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def database_scalar(sql):
|
||||
return command(
|
||||
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", sql, capture=True)
|
||||
|
||||
|
||||
def recreate(service, fault_name=None):
|
||||
environment = os.environ.copy()
|
||||
environment["FAULT_CRASH_AFTER_OUTBOX_COMMIT"] = "false"
|
||||
environment["FAULT_CRASH_AFTER_RESULT_COMMIT"] = "false"
|
||||
if fault_name:
|
||||
environment[fault_name] = "true"
|
||||
command("up", "-d", "--no-deps", "--force-recreate", service,
|
||||
environment=environment)
|
||||
|
||||
|
||||
def wait_http(client, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API did not become ready")
|
||||
|
||||
|
||||
def wait_job(attachment_id, expected="ready", seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
status = database_scalar(
|
||||
"SELECT status FROM attachment_jobs WHERE attachment_id = '"
|
||||
+ attachment_id + "';")
|
||||
if status == expected:
|
||||
return
|
||||
if status == "failed":
|
||||
raise Failure("fault-injected job reached failed")
|
||||
time.sleep(1)
|
||||
raise Failure("fault-injected job did not reach " + expected)
|
||||
|
||||
|
||||
def latest_attachment(chat_id, fault_column):
|
||||
row = database_scalar(
|
||||
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
|
||||
"WHERE a.chat_id = '" + chat_id + "' AND j." + fault_column
|
||||
+ " = true ORDER BY a.created_at DESC LIMIT 1;")
|
||||
check(bool(row), "fault marker was not persisted")
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
client = Client("http://localhost:8080")
|
||||
wait_http(client)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
original = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
# Crash exactly after attachment + outbox COMMIT, before the request thread publishes.
|
||||
recreate("api", "FAULT_CRASH_AFTER_OUTBOX_COMMIT")
|
||||
wait_http(client)
|
||||
try:
|
||||
client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=outbox.png",
|
||||
202, actor=alice, raw=original, headers={"Content-Type": "image/png"},
|
||||
schema="Attachment")
|
||||
raise Failure("outbox fault did not terminate the API request")
|
||||
except Failure as error:
|
||||
if "outbox fault did not" in str(error):
|
||||
raise
|
||||
|
||||
outbox_attachment = latest_attachment(chat["id"], "publish_fault_injected")
|
||||
wait_job(outbox_attachment)
|
||||
recreate("api")
|
||||
wait_http(client)
|
||||
recovered = client.call(
|
||||
"GET", "/api/v1/attachments/" + outbox_attachment,
|
||||
actor=bob, schema="Attachment")["json"]
|
||||
check(recovered["status"] == "ready", "outbox recovery did not publish the job")
|
||||
check(len(recovered["variants"]) == 3, "outbox recovery did not finish all variants")
|
||||
|
||||
# Crash after S3 effects + result COMMIT, before RabbitMQ consumer ack.
|
||||
recreate("worker", "FAULT_CRASH_AFTER_RESULT_COMMIT")
|
||||
queued = client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=ack-window.png",
|
||||
202, actor=alice, raw=original, headers={"Content-Type": "image/png"},
|
||||
schema="Attachment")["json"]
|
||||
wait_job(queued["id"])
|
||||
ready = client.wait_ready(bob, queued, wait_seconds=120)
|
||||
check(len(ready["variants"]) == 3, "result crash lost variants")
|
||||
result_marker = database_scalar(
|
||||
"SELECT result_fault_injected FROM attachment_jobs WHERE attachment_id = '"
|
||||
+ queued["id"] + "';")
|
||||
check(result_marker == "t", "result commit fault marker was not persisted")
|
||||
recreate("worker")
|
||||
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
queues = command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues",
|
||||
"name", "messages_ready", "messages_unacknowledged", capture=True)
|
||||
if "attachments.process.level4.v1\t0\t0" in queues:
|
||||
break
|
||||
time.sleep(1)
|
||||
variant_count = database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id = '"
|
||||
+ queued["id"] + "';")
|
||||
check(variant_count == "3", "redelivery duplicated committed variants")
|
||||
print("PASS: outbox commit/publish crash and result commit/ack crash recovered")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise durable queued upload, component restarts and duplicate delivery.
|
||||
|
||||
Run from the repository root against the local compose.yaml:
|
||||
python student_tests/lab5_worker_recovery.py
|
||||
The script preserves volumes and leaves the complete stack running.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
from image_probe import jpeg_size # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env.example"]
|
||||
|
||||
|
||||
def command(*args, capture=False):
|
||||
result = subprocess.run(
|
||||
[*COMPOSE, *args], cwd=ROOT, check=True,
|
||||
text=True, capture_output=capture)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def database_scalar(sql):
|
||||
return command(
|
||||
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", sql, capture=True)
|
||||
|
||||
|
||||
def env_value(name):
|
||||
for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith(name + "="):
|
||||
return line.split("=", 1)[1]
|
||||
raise Failure("missing " + name + " in .env.example")
|
||||
|
||||
|
||||
def wait_http_ready(client, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API did not become ready after restart")
|
||||
|
||||
|
||||
def wait_rabbit(seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
result = subprocess.run(
|
||||
[*COMPOSE, "exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping"],
|
||||
cwd=ROOT, text=True, capture_output=True)
|
||||
if result.returncode == 0:
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure("RabbitMQ did not become ready after restart")
|
||||
|
||||
|
||||
def main():
|
||||
client = Client("http://localhost:8080")
|
||||
command("stop", "worker")
|
||||
try:
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload did not remain queued")
|
||||
client.call(
|
||||
"GET", "/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
409, actor=bob)
|
||||
linked = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
check(linked["attachment_ids"] == [attachment["id"]],
|
||||
"queued attachment could not be linked")
|
||||
|
||||
command("restart", "api")
|
||||
command("restart", "rabbitmq")
|
||||
wait_rabbit()
|
||||
wait_http_ready(client)
|
||||
command("start", "worker")
|
||||
|
||||
ready = client.wait_ready(bob, attachment, wait_seconds=120)
|
||||
check(ready["status"] == "ready", "attachment did not become ready")
|
||||
downloaded = client.call(
|
||||
"GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
||||
check(downloaded["bytes"] == original, "original bytes changed")
|
||||
variants = {item["name"]: item for item in ready["variants"]}
|
||||
check(set(variants) == {"thumbnail", "crop", "optimized"},
|
||||
"expected thumbnail, crop and optimized variants")
|
||||
check((variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160),
|
||||
"thumbnail dimensions differ from contract")
|
||||
check((variants["crop"]["width"], variants["crop"]["height"]) == (128, 128),
|
||||
"crop dimensions differ from contract")
|
||||
crop = client.call(
|
||||
"GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=crop",
|
||||
actor=bob)
|
||||
check(crop["headers"].get_content_type() == "image/jpeg", "crop is not JPEG")
|
||||
check(jpeg_size(crop["bytes"]) == (128, 128), "crop JPEG has wrong dimensions")
|
||||
optimized = client.call(
|
||||
"GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=optimized",
|
||||
actor=bob)
|
||||
check(optimized["headers"].get_content_type() == "image/webp", "optimized is not WebP")
|
||||
check(optimized["bytes"][:4] == b"RIFF" and optimized["bytes"][8:12] == b"WEBP",
|
||||
"optimized payload has no WebP signature")
|
||||
|
||||
job_row = database_scalar(
|
||||
"SELECT id || '|' || pipeline_version || '|' || attempt || '|' || "
|
||||
"to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') "
|
||||
"FROM attachment_jobs WHERE attachment_id = '" + attachment["id"] + "';")
|
||||
job_id, pipeline_version, attempt, created_at = job_row.split("|")
|
||||
payload = json.dumps({
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": attachment["id"],
|
||||
"pipeline_version": int(pipeline_version),
|
||||
"attempt": int(attempt),
|
||||
"created_at": created_at,
|
||||
}, separators=(",", ":"))
|
||||
command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqadmin",
|
||||
"-u", env_value("RABBITMQ_USER"),
|
||||
"-p", env_value("RABBITMQ_PASSWORD"),
|
||||
"publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"),
|
||||
"routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"),
|
||||
"payload=" + payload,
|
||||
'properties={"delivery_mode":2,"content_type":"application/json"}')
|
||||
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
queued = command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues",
|
||||
"name", "messages_ready", "messages_unacknowledged", capture=True)
|
||||
if env_value("ATTACHMENT_JOB_QUEUE") + "\t0\t0" in queued:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
variant_count = database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id = '"
|
||||
+ attachment["id"] + "';")
|
||||
status = database_scalar(
|
||||
"SELECT status FROM attachments WHERE id = '" + attachment["id"] + "';")
|
||||
check(variant_count == "3", "duplicate delivery changed the three variant rows")
|
||||
check(status == "ready", "duplicate delivery changed terminal ready state")
|
||||
check(hashlib.sha256(downloaded["bytes"]).hexdigest() == attachment["sha256"],
|
||||
"original checksum changed")
|
||||
print("PASS: stopped worker, durable restart recovery and duplicate delivery")
|
||||
finally:
|
||||
command("start", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user