From be2abbb7344ca6527066a2f6aa52159740b20e27 Mon Sep 17 00:00:00 2001 From: Mraksimus Date: Thu, 19 Nov 2026 16:48:52 +0300 Subject: [PATCH] Completed for a grade of four --- Dockerfile | 2 +- REPORT.md | 174 ++++------ build.gradle.kts | 1 + compose.yaml | 4 +- evidence/attachment-failover.txt | 4 +- evidence/compose.txt | 24 +- evidence/gradle-tests.txt | 8 +- evidence/level4-recovery.txt | 5 + evidence/persistence.txt | 34 +- evidence/s3-permissions.txt | 10 +- evidence/smoke.txt | 2 +- evidence/worker-recovery.txt | 6 +- scripts/attachment-level4-recovery.py | 187 +++++++++++ scripts/attachment-worker-recovery.py | 2 +- scripts/check-s3-permissions.sh | 2 +- .../kotlin/com/mtusi/messenger/main/App.kt | 5 + .../controllers/health/HealthController.kt | 8 +- .../models/attachment/AttachmentJobModel.kt | 2 +- .../attachment/AttachmentJobReplayModel.kt | 18 ++ .../models/attachment/AttachmentModel.kt | 1 + .../attachment/AttachmentOutboxModel.kt | 25 ++ .../attachment/AttachmentRepository.kt | 11 +- .../attachment/ExposedAttachmentRepository.kt | 53 +-- .../jobs/AttachmentJobRepository.kt | 42 ++- .../jobs/AttachmentOutboxRepository.kt | 33 ++ .../jobs/ExposedAttachmentJobRepository.kt | 297 +++++++++++------ .../jobs/ExposedAttachmentOutboxRepository.kt | 110 +++++++ .../attachment/AttachmentCleanupService.kt | 22 +- .../attachment/AttachmentImageProcessor.kt | 137 ++++++++ .../DefaultAttachmentProcessingService.kt | 87 ++--- .../attachment/DefaultAttachmentService.kt | 54 ++-- .../DefaultImageProcessingService.kt | 301 +++++++++++++----- .../services/attachment/ImageProcess.kt | 64 ++++ .../attachment/ImageProcessingService.kt | 4 +- .../jobs/AttachmentOutboxDispatchService.kt | 49 +++ .../jobs/LegacyAttachmentDispatchService.kt | 28 -- .../storage/ObjectStorageService.kt | 10 +- .../storage/S3ObjectStorageService.kt | 38 ++- .../main/operator/AttachmentJobsCommand.kt | 105 ++++++ .../main/plugins/AttachmentWorker.kt | 12 +- .../mtusi/messenger/main/plugins/Database.kt | 6 +- .../com/mtusi/messenger/main/worker/Worker.kt | 2 +- .../migrations/V10__attachment_job_replay.sql | 13 + .../V11__complete_attachment_variants.sql | 36 +++ .../migrations/V9__attachment_outbox.sql | 23 ++ .../attachment/AttachmentControllerTest.kt | 10 +- .../attachment/AttachmentCleanupTest.kt | 19 +- .../attachment/AttachmentPipelineTest.kt | 139 +++++++- .../attachment/ImageProcessingServiceTest.kt | 130 ++++++++ .../LegacyAttachmentMigrationTest.kt | 4 +- 50 files changed, 1831 insertions(+), 532 deletions(-) create mode 100644 evidence/level4-recovery.txt create mode 100644 scripts/attachment-level4-recovery.py create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobReplayModel.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentOutboxModel.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentOutboxRepository.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentOutboxRepository.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentImageProcessor.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcess.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/AttachmentOutboxDispatchService.kt delete mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/operator/AttachmentJobsCommand.kt create mode 100644 src/main/resources/migrations/V10__attachment_job_replay.sql create mode 100644 src/main/resources/migrations/V11__complete_attachment_variants.sql create mode 100644 src/main/resources/migrations/V9__attachment_outbox.sql diff --git a/Dockerfile b/Dockerfile index 2f60fd5f..1b618079 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ FROM eclipse-temurin:21-jre-alpine # Distinguishes builds of the same code, e.g. the image rolled out by scripts/rolling-update.sh. ARG BUILD_ID=dev LABEL org.opencontainers.image.revision=$BUILD_ID -RUN apk add --no-cache wget && addgroup -S app && adduser -S -G app app +RUN apk add --no-cache wget libwebp-tools && addgroup -S app && adduser -S -G app app WORKDIR /app COPY --from=build /src/build/libs/messenger.jar /app/messenger.jar USER app diff --git a/REPORT.md b/REPORT.md index f0cbd128..2795f0bb 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,15 +1,13 @@ # Отчёт — лабораторная 5 -- Автор, группа: Кучерявенко Максим, БВТ2303. -- Целевая оценка: **3**. Уровни 4 и 5 не заявляются. -- Основа: собственный проект lab4 на Kotlin 2.3.20, Ktor 3.3.0 и JDK 21. Архитектура controller → service → repository, Koin/KSP и Exposed 1.0.0 сохранены; стиль сверяется с существующими репозиториями проекта и Pixefy. -- Новые зависимости: RabbitMQ Java Client 5.26.0 и metadata-extractor 2.19.0. Библиотека Java Client предоставляет AMQP API, поэтому здесь используются её Java configuration objects; операции PostgreSQL написаны на Exposed DSL, S3 — на AWS SDK for Kotlin DSL. -- Commit/PR: после просмотра и публикации. -- ИИ использован как парный разработчик при реализации, ревью, подготовке тестов и отчёта. +- Автор, группа — Кучерявенко Максим, БВТ2303. +- Целевая оценка — **4**. +- Основа — Kotlin 2.3.20, Ktor 3.3.0, JDK 21, Koin и Exposed 1.0.0. Из предыдущих работ сохранены авторизация, чаты и сообщения, балансировка, идемпотентность отправки и приватное хранение оригиналов. +- ИИ использован для реализации, ревью, тестов и подготовки отчёта. -## Запуск из чистого clone +## Запуск -Нужны Docker с Compose v2, Python 3.9+, openssl. JVM-тесты требуют JDK 21 и работающий Docker: Testcontainers поднимает отдельные PostgreSQL, Redis, MinIO и RabbitMQ. +Для стенда нужны Docker с Compose v2, Python 3.9+ и OpenSSL. Для локального прогона JVM-тестов нужны JDK 21, Docker и `cwebp` с `dwebp` в PATH. На macOS эти утилиты входят в пакет `webp` Homebrew. Образ приложения содержит `libwebp-tools`. ```sh ./scripts/init-secrets.sh @@ -18,149 +16,91 @@ export BASE_URL=http://localhost:8085 docker compose config --quiet docker compose up -d --build --wait -docker compose ps - make check ./gradlew test make test - -# Основной сценарий lab5: worker остановлен, 202 принят, -# обе API и broker перезапущены, тот же файл доходит до ready. python3 scripts/attachment-worker-recovery.py -# Доступ через другую API-реплику и приватность объекта. +# Сценарии остановки сервисов запускайте на локальном учебном стенде. +python3 scripts/attachment-level4-recovery.py python3 scripts/attachment-failover.py - -# Сохранность после перезапуска всех компонентов, включая worker/broker. make test-persistence ACTION_SCRIPT=scripts/restart.sh - -# Безопасная остановка с сохранением volumes: -docker compose down -# Только для намеренного удаления всех учебных данных lab5: -# docker compose down -v ``` -Секреты генерируются локально, существующие файлы сохраняются. В Git не включаются `.env`, `secrets/`, `run/development/`, данные volumes и build/cache. Проект Compose называется `mtusi-messenger-lab5`: его контейнеры, сеть и volumes отделены от lab4. Наружу опубликован только HAProxy на loopback; PostgreSQL, Redis, S3 и RabbitMQ доступны внутри сети стенда. +Flyway применяет V1–V11 до запуска API и worker. HAProxy публикует только loopback-порт. API, worker, PostgreSQL, Redis, RabbitMQ и MinIO общаются по внутренней сети Compose. Для PostgreSQL, RabbitMQ и MinIO выделены именованные volumes. Повторный запуск bootstrap сохраняет bucket приватным. Секреты генерируются в `secrets/` и монтируются как файлы. `docker compose down` сохраняет volumes. -Для запуска через `./gradlew run` нужны отдельные локальные настройки подключений в `run/development/config`. Шаблон не является готовой конфигурацией существующего PostgreSQL на компьютере; роль `username` необходимо заменить реальной. Автоматически менять локальную БД или её пользователей задача `run` не будет. Воспроизводимый путь проверки этой работы — Compose. +## Что сохраняется к моменту 202 -## Сервисы и настройки +API проверяет доступ к чату, имя, MIME, размер и корректность PNG/JPEG. Тело загружается с лимитом 10 MiB и общим таймаутом чтения 15 секунд. Проверка заголовка ограничивает изображение 40 миллионами пикселей до декодирования. Полная валидация перед 202 сохраняет поведение lab4 для битых изображений. -| Роль | Compose service | Версия / хранение | -| --- | --- | --- | -| Балансировщик | lb | HAProxy 3.2.4; внешний порт LAB_HTTP_PORT, по умолчанию 8080 | -| HTTP API | api × 2 | Образ mtusi-messenger:lab5, собранный из исходников | -| Обработчик | worker | Тот же образ, отдельный процесс с командой `worker` | -| Метаданные и задания | postgres | PostgreSQL 17.6; postgres-data | -| Кэш сессий | redis | Redis 8.4.0; без persistence, источник истины — PostgreSQL | -| Объектное хранилище | s3 | MinIO RELEASE.2025-09-07T16-13-09Z; s3-data | -| Durable очередь | broker | RabbitMQ 4.1.8-alpine; broker-data, стабильный hostname broker | +1. API создаёт намерение в `attachment_uploads` и записывает исходные байты в приватный `staging/` в MinIO. +2. Одна транзакция PostgreSQL создаёт метаданные queued, строку `attachment_jobs` и запись `attachment_outbox`, затем закрывает намерение загрузки. +3. После COMMIT API отвечает `202` со снимком queued. За загрузку отвечает API, за доставку задания в RabbitMQ отвечает процесс worker. -MinIO и mc закреплены digest из lab4. RabbitMQ также закреплён multi-platform digest `sha256:cd4009416f83d7e5895020cd71b0963f3b8a4dfc3de74bbb6fb3bfe5f81f01e0`. Одноразовый `migrate` применяет миграции Flyway V1–V8 до старта API и worker. `s3-init` создаёт приватный bucket и policy и допускает повторный запуск. +Фоновый диспетчер выбирает готовую строку outbox через `FOR UPDATE SKIP LOCKED` и фиксирует аренду на 30 секунд. Затем вне транзакции публикует persistent JSON с mandatory routing и ждёт publisher confirm. После подтверждения удаляет строку с проверкой токена аренды. При отказе публикации строка снова доступна через секунду. После падения диспетчера её подберёт следующий процесс по истечении аренды. Повторная публикация между confirm и удалением строки допустима. -Переменные `.env.example` сопоставлены реализации так: +Если RabbitMQ остановлен, API остаётся ready при доступных PostgreSQL и MinIO. Подтверждённая загрузка ждёт в outbox. Worker начинает работу после восстановления RabbitMQ. В очереди хранятся идентификаторы и служебные поля из `contracts/jobs.schema.json`. -- `API_REPLICAS`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS`, `S3_BUCKET` и `LAB_HTTP_PORT` управляют существующим стендом. -- PostgreSQL и Redis используют `DB_*` и `REDIS_*` с password-файлами. `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `APP_ENV` и переключатель `SESSION_STORE` из skeleton не управляют выбранной реализацией. -- `BROKER_HOST`, `BROKER_PORT`, `BROKER_USER`, `BROKER_PASSWORD_FILE`, `BROKER_QUEUE` задают RabbitMQ. Вместо `BROKER_URL_FILE` используются отдельные поля, чтобы URL с credentials не попадал в логи. Имя очереди по умолчанию — `attachment.process.v1`. -- `WORKER_CONCURRENCY=1` задаёт число последовательных consumer-циклов worker; допустимы 1–4. Каждый цикл держит не более одной unacked delivery. Проверенный профиль использует 1; увеличение требует отдельной оценки памяти. -- `JOB_MAX_ATTEMPTS=5` ограничивает количество захватов работы, `JOB_TIMEOUT_SECONDS=60` задаёт coroutine timeout, lease устанавливается на timeout + 15 секунд. -- `STAGING_RETENTION_SECONDS=86400` задаёт задержку удаления staging для terminal-состояний; минимум 3600. Для queued/processing исходник не удаляется по возрасту. -- Регион S3 фиксирован `us-east-1`; пределы 10 MiB и 40 млн пикселей закреплены в коде и не увеличиваются переменными окружения. +## Worker и варианты -## Архитектура и граница 202 +RabbitMQ хранит durable classic очередь в volume. Каждый цикл worker последовательно берёт одну manual-ack доставку через `basicGet`, поэтому в работе не более одной доставки на цикл. Параллелизм по умолчанию 1, конфигурация допускает 1–4. Декодирование сериализовано внутри процесса worker. -Клиент → HAProxy → API → PostgreSQL + приватный staging S3 → RabbitMQ → worker → final S3 + PostgreSQL. +Worker проверяет размер и SHA-256 исходника. Дочерняя JVM декодирует его один раз и создаёт все варианты. Она ограничена heap 384 MiB и дедлайном 25 секунд. По дедлайну родитель завершает процесс и его потомков. `cwebp` получает уже уменьшенный растр и имеет отдельный лимит 15 секунд. Основная JVM worker ограничена heap 320 MiB, контейнер — 1 GiB памяти и 1 CPU. Таймаут задания по умолчанию 60 секунд, аренда — таймаут плюс 15 секунд. -API принимает весь файл, проверяет доступ к чату, filename, MIME, размер и корректность изображения. HTTP-тело читается порциями по 8192 bytes во временный файл с лимитом 10 MiB и таймаутом 15 секунд; семафор допускает одну активную загрузку на API-реплику. Для сохранения обязательных проверок lab4 перед 202 выполняется полная валидация ImageIO, включая ограничения пикселей, CRC/IEND PNG и EOI JPEG. Resize и публикация вариантов выполняются только worker. +| Вариант | Результат | +| --- | --- | +| original | Исходные байты и MIME, ключ `originals/` | +| thumbnail | JPEG, вписанный в 256×256 без увеличения, для 320×200 получается 256×160 | +| crop | JPEG, центральный квадрат 128×128. Маленькие изображения при необходимости увеличиваются | +| optimized | WebP, вписанный в 1280×1280 без увеличения | -Далее API: +Для thumbnail и optimized размеры округляются вниз до целого пикселя, минимум 1. EXIF orientation 1–8 учитывается для всех вариантов. JPEG использует качество 0,85 и белый фон под прозрачностью. WebP кодируется `cwebp` с параметрами `-q 82 -m 4 -alpha_q 100 -metadata none` и сохраняет альфа-канал. Варианты строятся из отрисованного растра, поэтому EXIF и другие метаданные исходника не копируются. -1. Регистрирует намерение загрузки в унаследованном журнале `attachment_uploads`. -2. Сохраняет неизменённые bytes в `staging/` в MinIO. -3. В одной DB-транзакции создаёт Attachment со статусом queued и строку `attachment_jobs`, затем удаляет завершённое намерение из журнала. -4. Публикует persistent JSON в durable очередь и ожидает publisher confirm. -5. Возвращает **202** со снимком queued, пустыми variants и error_code=null. +Worker записывает оригинал и три варианта по детерминированным ключам. Затем одна транзакция публикует метаданные вариантов и статус ready под проверкой lease token. ACK следует после фиксации результата или устойчивого сохранения следующей попытки в outbox. Повтор завершённой работы подтверждается без обработки. Ключ результата — вложение, версия pipeline 1 и имя варианта. -Worker может успеть завершить работу до первого GET, но ответ POST остаётся снимком queued. Подтверждённая 202 работа уже имеет staging, метаданные, запись задания и подтверждённое брокером сообщение. Локальный временный файл после этого не нужен. +Готовые варианты читаются через авторизованный API. Для queued, processing и failed контент отвечает 409. Незнакомый вариант готового вложения и обращение постороннего к вложению возвращают 404. История сообщений и повторы по Idempotency-Key сохраняют прежнюю семантику. -При ошибке публикации 202 не возвращается. Если строка ещё queued, она переводится в failed с `enqueue_failed`; уже захваченная или завершённая работа не откатывается. После жёсткого crash между COMMIT и publish может остаться неподтверждённая queued-запись без сообщения: автоматическое восстановление этого окна относится к outbox уровня 4 и здесь не заявляется. +V8 сохраняет старые оригиналы lab4 и ставит их в очередь на обработку. V9 переносит ожидающие задачи в outbox. V11 ставит в очередь вложения уровня 3, у которых ещё нет crop и optimized. Эти работы читают оригинал по существующему ключу. На время генерации контент отвечает 409. -## Договор producer/worker и состояния +## Повторы и операторский replay -`AttachmentJob` соответствует обязательным полям `contracts/jobs.schema.json`: schema_version=1, job_id, type=attachment.process.v1, attachment_id, pipeline_version=1, attempt и created_at. В сообщении нет bytes, credentials, пользовательского пути или URL. Worker получает доверенные метаданные по ID из PostgreSQL. +Временные ошибки хранилища, таймауты и ошибки обработки повторяются с экспоненциальной задержкой и jitter в диапазоне от половины до полного интервала. Верхняя граница интервала начинается с 1 секунды и ограничена 30 секундами. PostgreSQL хранит причину и время следующей попытки. Retry и запись outbox фиксируются вместе. Лимит по умолчанию — пять попыток на запуск. Невалидное изображение и превышение лимитов исходника приводят к failed с безопасным кодом. -Логическая job_id стабильна. Broker redelivery сохраняет envelope. Управляемый retry публикует сообщение той же job с увеличенным attempt. Worker подтверждает старую доставку после подтверждения новой публикации. Если lease ещё действует, worker отправляет тот же envelope в конец очереди, сохраняя номер попытки. Если старый envelope приходит после изменения DB-счётчика, worker публикует актуальную попытку без повторной обработки. +Если доставка возвращается при действующей аренде, worker планирует её в outbox на момент истечения lease. Очередь продолжает обслуживать остальные задания. После истечения lease job получает новую попытку в пределах бюджета. Успешное завершение или запись следующей попытки по старому токену блокируются после смены аренды. -- queued → processing: worker блокирует запись задания, проверяет статус и lease, увеличивает счётчик попыток и записывает случайный token. -- processing → ready: после записи original и thumbnail одна DB-транзакция публикует вариант и состояние ready, затем снимает lease. -- processing → queued: при временной ошибке с оставшимися попытками PostgreSQL сохраняет внутренний код причины и время следующего запуска. Повтор можно запустить сразу, поэтому используется время БД в момент перехода. Затем worker публикует retry и ждёт подтверждения. -- processing → failed: постоянная ошибка изображения или исчерпание попыток. В API остаётся безопасный error_code, без stack trace и секретов. -- ready/failed + повторная delivery: no-op с ack. Старый token не может опубликовать результат после нового захвата. +Реестр failed хранится в PostgreSQL. Команды доступны пользователю с доступом к локальному Docker и базе: -Default consumer использует manual `basicGet`: следующий запрос выполняется после завершения предыдущего handler. Это явно ограничивает число delivery в работе; значение basicQos(1) не используется как единственная гарантия для polling API. Ack отправляется после устойчивого результата или подтверждённой передачи следующей попытки. При остановке/обрыве unacked delivery возвращается в очередь. Истёкший lease допускает новый захват; попытки по-прежнему ограничены DB-счётчиком. +```sh +docker compose run --rm --no-deps -T worker attachment-jobs failed +docker compose run --rm --no-deps -T worker attachment-jobs replay JOB_ID OPERATOR +``` -Это доставка **at least once** с идемпотентным эффектом, а не exactly-once. Объекты имеют детерминированные ключи, а вариант в БД — первичный ключ `(attachment_id, name)`. В этом этапе используется только pipeline_version=1. +Первая команда выводит до 100 последних отказов. Replay сохраняет ID задания, открывает новый бюджет попыток и пишет аудит с операторской меткой, прежним кодом ошибки, временем и номером попытки. Метка передаётся оператором и используется для аудита, предоставление доступа к команде остаётся на стороне Docker. Replay отклоняет постоянную ошибку изображения, успешно завершённую или выполняющуюся задачу и задачу после начала очистки. Поэтому повтор команды для уже восстановленного задания не запускает дубль. -## Worker и изображения +Уборщик удаляет осиротевшие исходники после grace period. Срок хранения staging для терминальных состояний по умолчанию 86400 секунд. У готового вложения сохраняются оригинал и варианты. У failed нового вложения по истечении срока удаляются исходник из staging и частичные результаты. У импортированных вложений сохраняется прежний оригинал. Очистка помечает строку перед удалением, чтобы replay и удаление исходника не могли начаться одновременно. -Worker запускается отдельным процессом и не принимает бизнес-HTTP-запросы. Служебные health endpoints нужны Compose. Остановка worker не переводит resize в API: новые вложения остаются queued, текстовые сообщения продолжают работать, queued-вложение разрешено прикреплять к сообщению. +## Результаты проверок -Worker проверяет размер и SHA-256 staging, один раз декодирует изображение и строит thumbnail. Для записи оригинала в S3 worker использует уже проверенные байты: +Контрольный прогон выполнен **18 ноября 2026 года** на отдельном Compose-стенде `mtusi-messenger-lab5-level4` с портом 8086. Все результаты получены на сборке уровня 4. -- `originals/` — исходные bytes без перекодирования; -- `variants//v1/thumbnail.jpg` — JPEG, вписанный в 256×256 без увеличения; размеры округляются вниз, минимум 1 px. +| Проверка | Результат | +| --- | --- | +| Сборка и JVM-тесты | `rtk gradlew compileKotlin compileTestKotlin test --console=plain` на JDK 21. Все **98 тестов** прошли. Покрыты backoff, истечение lease, повторная доставка после рестарта RabbitMQ, аудит replay, миграции, очистка, EXIF, декодирование WebP и завершение дочернего процесса по дедлайну. [Сводка JUnit](evidence/gradle-tests.txt) | +| Структура, стенд и smoke | `make check`, `docker compose config --quiet`, `docker compose up -d --build --wait` и `make test` прошли. [Структура](evidence/check.txt), [контейнеры и лимиты](evidence/compose.txt), [smoke](evidence/smoke.txt) | +| Падение producer после COMMIT | RabbitMQ и worker остановлены. API принял файл, запись outbox осталась после SIGKILL принявшей реплики. После восстановления тот же ID стал ready со всеми вариантами. [Лог проверки](evidence/level4-recovery.txt) | +| Падение worker после записи S3 | Одноразовый worker завершился с кодом 86 до фиксации результата и ACK. API ответил 409. После истечения аренды вторая попытка завершила то же вложение, сохранив оригинал и контрольные суммы всех трёх вариантов. [Лог проверки](evidence/level4-recovery.txt) | +| Реестр и replay | При недоступном MinIO задание исчерпало бюджет. CLI показал failed. После восстановления хранилища replay довёл то же задание до ready на второй попытке. Повторная команда получила отказ. [Лог проверки](evidence/level4-recovery.txt) | +| Перезапуск worker, API и RabbitMQ | Задание осталось queued при остановленном worker и сохранилось после перезапуска API и RabbitMQ. После старта worker доступны оригинал и thumbnail. [Лог восстановления](evidence/worker-recovery.txt) | +| Переключение API и приватность S3 | После остановки принявшей реплики другая вернула тот же оригинал. Чужой пользователь получил 404, анонимный S3 GET получил 403. Служебная учётная запись ограничена рабочими префиксами. [Переключение](evidence/attachment-failover.txt), [права S3](evidence/s3-permissions.txt) | +| Перезапуск всего стенда | Подтверждённые данные и действующая сессия сохранились. `StartedAt` изменился у всех восьми контейнеров. [Сохранность и времена запуска](evidence/persistence.txt) | -EXIF orientation 1–8 применяется при отрисовке. Alpha компонуется на белый. JPEG кодируется стандартным ImageIO writer; чужие metadata в новый JPEG не переносятся. Fixture 320×200 даёт thumbnail 256×160. Метаданные варианта содержат собственные MIME, размеры, размер bytes и SHA-256. +## Ограничения и источники -Для queued/processing/failed скачивание content возвращает 409. Для ready доступен original и thumbnail, отсутствующий вариант получает 404. Чужой пользователь получает 404 до выдачи метаданных или содержимого. Failed-вложение нельзя добавить в новое сообщение, но уже созданное сообщение корректно воспроизводится по прежнему Idempotency-Key, даже если вложение позднее стало failed. +PostgreSQL, MinIO, RabbitMQ и HAProxy работают в одном экземпляре. Проверяется сохранность при рестарте с volumes. Восстановление задачи после падения worker может занять время аренды и задержки новой попытки. API перед 202 по-прежнему полностью декодирует ограниченное изображение. Жёсткий таймаут дочернего декодера применяется к worker. -При обновлении с lab4 миграция V8 переводит вложения без thumbnail в queued и создаёт задания для уже сохранённых оригиналов. Worker снимает отметку об ожидающей публикации после подтверждения RabbitMQ. Если процесс остановится раньше, он повторит публикацию того же задания. Существующий объект читается по прежнему ключу и сохраняет исходные байты. Скачивание снова доступно после готовности thumbnail. +Семантика outbox и повторной доставки рассчитана на at least once. Схема защищает эффект в рамках pipeline версии 1. Итоговый размер WebP зависит от исходника и не обязан быть меньше размера маленького PNG или JPEG. -## Сохранность, cleanup и безопасность - -Очередь durable classic, сообщения persistent, публикации используют mandatory routing и publisher confirms. RabbitMQ сохраняет данные в отдельном volume. Producer confirm означает приём брокером, consumer ack — устойчивый эффект worker; эти подтверждения не взаимозаменяемы. - -Унаследованный журнал закрывает orphan-окно между staging PUT и записью метаданных. После часового grace period уборщик помечает просроченные намерения abandoned и удаляет связанные ключи. Записи сохраняются для повторного удаления позднего PUT. По истечении retention уборщик удаляет staging для ready и все записанные файлы для failed. У вложений из lab4 сохраняется оригинал. Отметка staging_cleaned фиксирует завершение уборки. Очистка обходит только terminal-состояния. - -Root S3 credentials доступны только MinIO/bootstrap. Прикладная policy дополнена разрешениями для staging/* и variants/*, сохраняя запрет административных операций и записи вне рабочих prefixes. S3 и console не опубликованы на host, console привязана к loopback внутри контейнера. Пароль RabbitMQ читается из secret-файла; брокер также не опубликован наружу. - -При завершении приложения явно отменяются consumer, cleanup и синхронизация revocation, затем закрываются подключения. Это предотвращает работу старых фоновых циклов после закрытия DI/БД и их вмешательство в другие экземпляры тестового приложения. - -## Проверки уровня 3 - -Стендовые проверки из таблицы выполнены **17 ноября 2026 года** на порту 8085. После исправлений **18 ноября** отдельно собран стенд на порту 8086, где повторно прошли публичный smoke и сценарий восстановления worker. Полный JVM-набор также запущен повторно после правок. - -| Проверка | Команда / сценарий | Результат и доказательство | -| --- | --- | --- | -| Структура и контракт | `make check` | PASS; [вывод](evidence/check.txt) | -| Конфигурация и запуск Compose | `docker compose config --quiet`, `docker compose up -d --build --wait` | Все восемь рабочих контейнеров healthy; [конфигурация и состояние стенда](evidence/compose.txt) | -| JVM-регрессии и новый pipeline | `rtk gradlew compileKotlin compileTestKotlin test --console=plain` | **94 теста, 0 ошибок и пропусков**, включая обновление с lab4, очистку частичного результата и обработку следующей задачи при действующем lease; [сводка по JUnit XML](evidence/gradle-tests.txt) | -| Публичный сценарий lab5 | `make test BASE_URL=http://localhost:8085` | PASS: upload 202, ready, оригинал и настоящий JPEG thumbnail; [вывод](evidence/smoke.txt) | -| Worker остановлен, API и broker перезапущены | `python3 scripts/attachment-worker-recovery.py` | 202 queued → сообщение создано → content 409 → restart обеих API и broker с volumes → запуск worker → та же запись ready, неизменённый оригинал и JPEG 256×160; [вывод с attachment_id](evidence/worker-recovery.txt) | -| Durable доставка и повтор одной job | `RabbitMqJobBrokerServiceTest`, `AttachmentPipelineTest` в JVM-наборе | Реальный restart контейнера RabbitMQ сохраняет подтверждённое сообщение; nack возвращает тот же envelope; повтор завершённой job сохраняет прежний результат | -| Ограниченные попытки и failed | `AttachmentPipelineTest` в JVM-наборе | Busy lease, постоянная ошибка изображения и исчерпание временных попыток; failed запрещает content и новое сообщение, прежний Idempotency-Key воспроизводит созданное сообщение | -| Правила thumbnail | `ImageProcessingServiceTest` в JVM-наборе | Шесть тестов: JPEG/PNG EXIF orientation, alpha на белом, размеры с округлением вниз, отсутствие увеличения и удаление orientation из результата | -| Переключение API-реплики | `python3 scripts/attachment-failover.py` | Загрузившая реплика остановлена, другая отдаёт оригинал с тем же SHA-256; анонимный S3 GET — 403, чужой пользователь — 404; [вывод](evidence/attachment-failover.txt) | -| Привилегии S3 и повторный bootstrap | `scripts/check-s3-permissions.sh` | Запрещены административные операции, изменение policy и запись вне рабочих prefixes; console недоступна из общей сети; [вывод](evidence/s3-permissions.txt) | -| Полный restart с сохранением данных | `make test-persistence BASE_URL=http://localhost:8085 ACTION_SCRIPT=scripts/restart.sh` | PASS: подтверждённые данные и сессия сохранены; StartedAt изменился у обеих API, worker, broker, LB, PostgreSQL, Redis и MinIO; [вывод и времена до/после](evidence/persistence.txt) | - -Для сценариев со стендом установлены `LAB_HTTP_PORT=8085` и `BASE_URL=http://localhost:8085`, как в инструкции запуска выше. Полный JVM-прогон также проверяет сохранённые возможности lab1–lab4: авторизацию, чаты, конкурентную запись, идемпотентность сообщений, межрепличный отзыв сессий, миграции и отказы зависимостей. Исторические материалы lab4 находятся в `archive/lab4/`; они не являются доказательствами lab5. - -## Ограничения и границы следующего этапа - -- Outbox/автоматическое восстановление разрыва COMMIT/publish, backoff с jitter, DLQ/operator replay, crop и optimized WebP относятся к уровню 4 и не реализованы. -- Валидация использует массив ≤10 MiB и полный raster ImageIO; это не полностью потоковый декодер. Очень широкие изображения остаются краевым случаем памяти исходной реализации. Worker ограничен 1 GiB контейнерной памяти; при JVM OOM включён аварийный выход с последующим restart, а число попыток сохраняется в БД. -- Coroutine timeout кооперативный и не гарантирует принудительное прерывание блокирующего декодера в середине вызова. Lease/token предотвращает публикацию устаревшим обработчиком, но не заменяет жёсткое ограничение CPU-времени отдельного декодера. -- PostgreSQL, MinIO, RabbitMQ и HAProxy одноузловые. Volumes сохраняют данные при restart; отказоустойчивость узла и backup не заявляются. -- После crash незавершённая processing-работа может ждать истечения lease. Секундного SLA на восстановление при любом сбое не заявляется. -- Сравнение нескольких pipeline versions и защита от перезаписи результата старой версией относятся к исследовательскому треку; сейчас применяется версия 1. - -## Источники - -- Задание, contracts/README.md, contracts/attachment-states.md и contracts/jobs.schema.json этого репозитория. -- [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability). -- [Publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms). -- [RabbitMQ Java Client API](https://www.rabbitmq.com/client-libraries/java-api-guide). +- [Контракт API](contracts/README.md) и [состояния вложений](contracts/attachment-states.md). +- [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability) и [publisher confirms / consumer ack](https://www.rabbitmq.com/docs/confirms). - [AWS SDK for Kotlin](https://github.com/awslabs/aws-sdk-kotlin). -- [metadata-extractor](https://github.com/drewnoakes/metadata-extractor). +- [Параметры cwebp](https://developers.google.com/speed/webp/docs/cwebp). diff --git a/build.gradle.kts b/build.gradle.kts index 2f3348ba..352567ac 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -86,6 +86,7 @@ sourceSets { tasks { test { useJUnitPlatform() + systemProperty("messenger.image.classpath", sourceSets["main"].runtimeClasspath.asPath) } shadowJar { diff --git a/compose.yaml b/compose.yaml index 6a3c3a67..bea573ce 100644 --- a/compose.yaml +++ b/compose.yaml @@ -165,8 +165,6 @@ services: condition: service_healthy s3-init: condition: service_completed_successfully - broker: - condition: service_healthy environment: APP_HOST: "0.0.0.0" APP_PORT: "8080" @@ -236,7 +234,7 @@ services: BROKER_USER: messenger BROKER_PASSWORD_FILE: /run/secrets/broker_password BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1} - JAVA_TOOL_OPTIONS: "-XX:+ExitOnOutOfMemoryError" + JAVA_TOOL_OPTIONS: "-Xmx320m -XX:+ExitOnOutOfMemoryError" WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt index 96df8612..9943eede 100644 --- a/evidence/attachment-failover.txt +++ b/evidence/attachment-failover.txt @@ -1,6 +1,6 @@ -Upload: 202 queued, then ready; instance=7a70d3b286da +Upload: 202 queued, then ready; instance=da6189641548 Anonymous S3 GET of known object: 403 Outsider metadata/content: 404 -Download: 200; instance=f000ea12f8bd +Download: 200; instance=b89ba5975865 SHA-256: 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 PASS: uploading replica stopped; original served by another replica; stopped replica restored diff --git a/evidence/compose.txt b/evidence/compose.txt index c89ebc6b..1bbd8b79 100644 --- a/evidence/compose.txt +++ b/evidence/compose.txt @@ -1,13 +1,11 @@ -$ docker compose config --quiet -Exit code: 0 - -$ docker compose ps -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "java -XX:MaxRAMPerc…" api 8 minutes ago Up 2 minutes (healthy) 8080/tcp -mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "java -XX:MaxRAMPerc…" api 8 minutes ago Up 2 minutes (healthy) 8080/tcp -mtusi-messenger-lab5-broker-1 rabbitmq@sha256:cd4009416f83d7e5895020cd71b0963f3b8a4dfc3de74bbb6fb3bfe5f81f01e0 "/bin/sh /bootstrap.…" broker 19 hours ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp -mtusi-messenger-lab5-lb-1 haproxy:3.2.4-alpine "docker-entrypoint.s…" lb 19 hours ago Up About a minute (healthy) 127.0.0.1:8085->8080/tcp -mtusi-messenger-lab5-postgres-1 postgres:17.6-alpine3.22 "docker-entrypoint.s…" postgres 19 hours ago Up 2 minutes (healthy) 5432/tcp -mtusi-messenger-lab5-redis-1 redis:8.4.0-alpine "docker-entrypoint.s…" redis 19 hours ago Up 2 minutes (healthy) 6379/tcp -mtusi-messenger-lab5-s3-1 quay.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e "/usr/bin/docker-ent…" s3 19 hours ago Up 2 minutes (healthy) 9000/tcp -mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "java -XX:MaxRAMPerc…" worker 8 minutes ago Up 2 minutes (healthy) 8080/tcp +api Up 10 seconds (healthy) 8080/tcp +api Up 10 seconds (healthy) 8080/tcp +broker Up 19 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +lb Up 6 seconds (healthy) 127.0.0.1:8086->8080/tcp +postgres Up 19 seconds (healthy) 5432/tcp +redis Up 19 seconds (healthy) 6379/tcp +s3 Up 19 seconds (healthy) 9000/tcp +worker Up 10 seconds (healthy) 8080/tcp +Worker memory bytes=1073741824; NanoCPUs=1000000000 +1.6.0 +libsharpyuv: 0.4.2 diff --git a/evidence/gradle-tests.txt b/evidence/gradle-tests.txt index 56fa37a2..57834776 100644 --- a/evidence/gradle-tests.txt +++ b/evidence/gradle-tests.txt @@ -1,6 +1,6 @@ Command: rtk gradlew compileKotlin compileTestKotlin test --console=plain (JDK 21) BUILD SUCCESSFUL -Tests: 94; failures: 0; errors: 0; skipped: 0 +Tests: 98; failures: 0; errors: 0; skipped: 0 com.mtusi.messenger.main.infrastructure.controllers.attachment.AttachmentControllerTest: tests=3 failures=0 com.mtusi.messenger.main.infrastructure.controllers.authentication.AuthenticationControllerTest: tests=9 failures=0 com.mtusi.messenger.main.infrastructure.controllers.chat.ChatControllerTest: tests=6 failures=0 @@ -22,12 +22,16 @@ com.mtusi.messenger.main.infrastructure.services.attachment.AttachmentPipelineTe busy redelivery lets the next attachment finish before lease expiration() worker distinguishes permanent image errors from bounded transient failures() stopped worker leaves a durable queued attachment and redelivery preserves its result() -com.mtusi.messenger.main.infrastructure.services.attachment.ImageProcessingServiceTest: tests=6 failures=0 +com.mtusi.messenger.main.infrastructure.services.attachment.ImageProcessingServiceTest: tests=10 failures=0 + worker subprocess preserves permanent image failures() PNG EXIF orientation is applied before resizing() transparent and semitransparent pixels are composited on white() invalid images and MIME mismatches are rejected before publishing a thumbnail() + published variants apply orientation and preserve the centered crop() missing or unknown EXIF orientation preserves the normal image() all JPEG EXIF orientations rotate or mirror pixels and strip orientation from the thumbnail() + optimized WebP preserves alpha and never enlarges a small source() + image subprocess stops at its deadline() thumbnail fits the box floors dimensions and never enlarges the source() com.mtusi.messenger.main.infrastructure.services.attachment.ImageValidatorTest: tests=3 failures=0 com.mtusi.messenger.main.infrastructure.services.attachment.LegacyAttachmentMigrationTest: tests=1 failures=0 diff --git a/evidence/level4-recovery.txt b/evidence/level4-recovery.txt new file mode 100644 index 00000000..a7873962 --- /dev/null +++ b/evidence/level4-recovery.txt @@ -0,0 +1,5 @@ +PASS: producer SIGKILL after PostgreSQL commit and before publish; outbox survived; id=dfeea134-7def-4f55-a601-825dc8a56208 +PASS: outbox recovered the same attachment with all three variants; id=dfeea134-7def-4f55-a601-825dc8a56208 +PASS: worker exited 86 after writing S3 objects, before result commit and ack; id=cf5b21c7-ecc8-4c43-8a28-a3f3761d1378 +PASS: redelivery recovered the same id, original and three variants on attempt 2; id=cf5b21c7-ecc8-4c43-8a28-a3f3761d1378 +PASS: failed job listed, replayed with same id on attempt 2, and replay refused after ready; id=99a5cbcd-5cde-482b-860c-c375d8a42af9 diff --git a/evidence/persistence.txt b/evidence/persistence.txt index a380f1d9..ec7c20e5 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,23 +1,23 @@ -python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8085" +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8086" Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. Before restart: -/mtusi-messenger-lab5-api-1 2026-11-17T11:48:25.6974278Z -/mtusi-messenger-lab5-api-2 2026-11-17T11:48:25.799164633Z -/mtusi-messenger-lab5-broker-1 2026-11-17T11:48:15.954373295Z -/mtusi-messenger-lab5-lb-1 2026-11-17T11:48:28.901775676Z -/mtusi-messenger-lab5-postgres-1 2026-11-17T11:48:15.943166503Z -/mtusi-messenger-lab5-redis-1 2026-11-17T11:48:15.95361167Z -/mtusi-messenger-lab5-s3-1 2026-11-17T11:48:15.94897392Z -/mtusi-messenger-lab5-worker-1 2026-11-17T11:48:25.696508175Z +/mtusi-messenger-lab5-level4-api-1 2026-11-18T11:08:43.347539047Z +/mtusi-messenger-lab5-level4-api-2 2026-11-18T11:08:17.102104132Z +/mtusi-messenger-lab5-level4-broker-1 2026-11-18T11:08:13.123424672Z +/mtusi-messenger-lab5-level4-lb-1 2026-11-18T11:05:22.404306926Z +/mtusi-messenger-lab5-level4-postgres-1 2026-11-18T11:05:06.926939794Z +/mtusi-messenger-lab5-level4-redis-1 2026-11-18T11:05:06.922586211Z +/mtusi-messenger-lab5-level4-s3-1 2026-11-18T11:07:45.175138173Z +/mtusi-messenger-lab5-level4-worker-1 2026-11-18T11:08:24.920627553Z After restart: -/mtusi-messenger-lab5-api-1 2026-11-17T11:52:14.708903753Z -/mtusi-messenger-lab5-api-2 2026-11-17T11:52:14.820903586Z -/mtusi-messenger-lab5-broker-1 2026-11-17T11:52:04.990389887Z -/mtusi-messenger-lab5-lb-1 2026-11-17T11:52:17.919454796Z -/mtusi-messenger-lab5-postgres-1 2026-11-17T11:52:04.990403929Z -/mtusi-messenger-lab5-redis-1 2026-11-17T11:52:04.984496471Z -/mtusi-messenger-lab5-s3-1 2026-11-17T11:52:04.992642512Z -/mtusi-messenger-lab5-worker-1 2026-11-17T11:52:14.708902794Z +/mtusi-messenger-lab5-level4-api-1 2026-11-18T11:09:23.075147802Z +/mtusi-messenger-lab5-level4-api-2 2026-11-18T11:09:22.927327177Z +/mtusi-messenger-lab5-level4-broker-1 2026-11-18T11:09:13.628215922Z +/mtusi-messenger-lab5-level4-lb-1 2026-11-18T11:09:26.709695053Z +/mtusi-messenger-lab5-level4-postgres-1 2026-11-18T11:09:13.629343672Z +/mtusi-messenger-lab5-level4-redis-1 2026-11-18T11:09:13.621602214Z +/mtusi-messenger-lab5-level4-s3-1 2026-11-18T11:09:13.629312422Z +/mtusi-messenger-lab5-level4-worker-1 2026-11-18T11:09:22.928541802Z diff --git a/evidence/s3-permissions.txt b/evidence/s3-permissions.txt index 59841603..ca044e59 100644 --- a/evidence/s3-permissions.txt +++ b/evidence/s3-permissions.txt @@ -1,10 +1,10 @@ - Container mtusi-messenger-lab5-s3-1 Running + Container mtusi-messenger-lab5-level4-s3-1 Running Bucket created successfully `local/messenger-attachments`. Access permission for `local/messenger-attachments` is set to `private` - Container mtusi-messenger-lab5-s3-1 Running + Container mtusi-messenger-lab5-level4-s3-1 Running Bucket created successfully `local/messenger-attachments`. Access permission for `local/messenger-attachments` is set to `private` - Container mtusi-messenger-lab5-s3-1 Running -PASS: app cannot administer MinIO, create buckets, change policy or write outside originals/ -wget: can't connect to remote host (172.26.0.2): Connection refused + Container mtusi-messenger-lab5-level4-s3-1 Running +PASS: app cannot administer MinIO, create buckets, change policy or write to an unrelated prefix +wget: can't connect to remote host (172.27.0.4): Connection refused PASS: console unavailable from shared network diff --git a/evidence/smoke.txt b/evidence/smoke.txt index 2448312f..cbe5f5a5 100644 --- a/evidence/smoke.txt +++ b/evidence/smoke.txt @@ -1,2 +1,2 @@ -python3 tests/smoke.py --base-url "http://localhost:8085" +python3 tests/smoke.py --base-url "http://localhost:8086" PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/worker-recovery.txt b/evidence/worker-recovery.txt index bcac0aeb..61d37898 100644 --- a/evidence/worker-recovery.txt +++ b/evidence/worker-recovery.txt @@ -1,3 +1,3 @@ -PASS: worker stopped; upload 202 queued; message accepted; content 409; attachment_id=0305bfe5-e30e-47cc-b81e-4f2c578273a6 -PASS: both APIs and broker restarted with existing volumes; same attachment remains queued; attachment_id=0305bfe5-e30e-47cc-b81e-4f2c578273a6 -PASS: same accepted job reached ready; original unchanged; thumbnail JPEG 256x160 with matching checksum; attachment_id=0305bfe5-e30e-47cc-b81e-4f2c578273a6 +PASS: worker stopped; upload 202 queued; message accepted; content 409; attachment_id=5fba0009-e3d4-4c96-b77d-472ead517cda +PASS: both APIs and broker restarted with existing volumes; same attachment remains queued; attachment_id=5fba0009-e3d4-4c96-b77d-472ead517cda +PASS: same accepted job reached ready; original unchanged; thumbnail JPEG 256x160 with matching checksum; attachment_id=5fba0009-e3d4-4c96-b77d-472ead517cda diff --git a/scripts/attachment-level4-recovery.py b/scripts/attachment-level4-recovery.py new file mode 100644 index 00000000..3ae5953d --- /dev/null +++ b/scripts/attachment-level4-recovery.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Prove the producer and worker crash boundaries on a local Compose stand.""" +import hashlib +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check +from image_probe import jpeg_size + +FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def docker(*args, timeout=45, expected=0): + result = subprocess.run( + ["docker", *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + check(result.returncode == expected, f"docker {args[0]} returned {result.returncode}, expected {expected}") + return (result.stdout if expected == 0 else result.stdout + result.stderr).strip() + + +def compose(*args, **kwargs): + return docker("compose", *args, **kwargs) + + +def sql(query): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query) + + +def metadata_state(attachment_id): + return sql( + "SELECT a.status, j.attempts, EXISTS " + "(SELECT 1 FROM attachment_outbox o WHERE o.job_id = j.id) " + "FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id " + f"WHERE a.id = '{uuid.UUID(attachment_id)}'" + ) + + +def await_api(client): + deadline = time.monotonic() + 30 + while True: + try: + client.call("GET", "/health/ready") + return + except Failure: + check(time.monotonic() < deadline, "API did not recover") + time.sleep(0.25) + + +def upload(client, actor, chat): + return client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, actor=actor, raw=FIXTURE, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + + +def verify_result(client, actor, queued): + attachment = client.wait_ready(actor, queued, wait_seconds=75) + path = f"/api/v1/attachments/{queued['id']}/content" + original = client.call("GET", path, actor=actor)["bytes"] + check(original == FIXTURE, "original bytes changed") + + variants = {variant["name"]: variant for variant in attachment["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "variant set differs") + for name, variant in variants.items(): + response = client.call("GET", f"{path}?variant={name}", actor=actor) + data = response["bytes"] + check(hashlib.sha256(data).hexdigest() == variant["sha256"], f"{name} checksum differs") + check(len(data) == variant["size_bytes"], f"{name} size differs") + check(response["headers"].get_content_type() == variant["content_type"], f"{name} MIME differs") + if name == "optimized": + check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP") + check((variant["width"], variant["height"]) == (320, 200), "optimized dimensions differ") + else: + expected = (128, 128) if name == "crop" else (256, 160) + check(jpeg_size(data) == expected, f"{name} dimensions differ") + + +def producer_crash(client, actor, reader, chat): + compose("stop", "worker") + compose("stop", "broker") + try: + result = upload(client, actor, chat) + queued = result["json"] + check(metadata_state(queued["id"]) == "queued|0|t", "outbox did not commit with the upload") + + instance = result["headers"]["X-Instance-Id"] + matches = [cid for cid in compose("ps", "-q", "api").splitlines() if cid.startswith(instance)] + check(len(matches) == 1, "uploading API replica was not identified") + replica = matches[0] + before = docker("inspect", "-f", "{{.State.StartedAt}}", replica) + docker("kill", "--signal", "KILL", replica) + docker("start", replica) + await_api(client) + check(before != docker("inspect", "-f", "{{.State.StartedAt}}", replica), "uploading API did not restart") + check(metadata_state(queued["id"]) == "queued|0|t", "pending outbox lost after API crash") + print(f"PASS: producer SIGKILL after PostgreSQL commit and before publish; outbox survived; id={queued['id']}", flush=True) + + compose("start", "broker", "worker") + verify_result(client, reader, queued) + print(f"PASS: outbox recovered the same attachment with all three variants; id={queued['id']}", flush=True) + finally: + compose("start", "broker", "api", "worker") + + +def worker_crash(client, actor, reader, chat): + compose("stop", "worker") + temporary = "lab5-worker-fault-" + uuid.uuid4().hex[:12] + try: + queued = upload(client, actor, chat)["json"] + compose( + "run", "--rm", "--no-deps", "-T", "--name", temporary, + "-e", f"LAB5_TEST_CRASH_AFTER_OBJECTS={queued['id']}", + "-e", "JOB_TIMEOUT_SECONDS=15", "worker", + timeout=70, expected=86, + ) + check(metadata_state(queued["id"]).startswith("processing|1|"), "worker did not stop before committing results") + client.call("GET", f"/api/v1/attachments/{queued['id']}/content", 409, actor=reader) + print(f"PASS: worker exited 86 after writing S3 objects, before result commit and ack; id={queued['id']}", flush=True) + + compose("start", "worker") + verify_result(client, reader, queued) + check(metadata_state(queued["id"]).startswith("ready|2|"), "recovery did not use exactly one new attempt") + print(f"PASS: redelivery recovered the same id, original and three variants on attempt 2; id={queued['id']}", flush=True) + finally: + subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT) + compose("start", "worker") + + +def operator_replay(client, actor, reader, chat): + compose("stop", "worker") + temporary = "lab5-worker-replay-" + uuid.uuid4().hex[:12] + try: + queued = upload(client, actor, chat)["json"] + job = sql(f"SELECT id FROM attachment_jobs WHERE attachment_id = '{uuid.UUID(queued['id'])}'") + job = str(uuid.UUID(job)) + compose("stop", "s3") + compose( + "run", "-d", "--rm", "--no-deps", "--name", temporary, + "-e", "JOB_MAX_ATTEMPTS=1", "worker", + ) + + deadline = time.monotonic() + 40 + while not metadata_state(queued["id"]).startswith("failed|1|"): + check(time.monotonic() < deadline, "one-attempt worker did not record the storage failure") + time.sleep(0.25) + + docker("stop", "--time", "6", temporary) + compose("up", "-d", "--wait", "s3") + await_api(client) + + command = ("run", "--rm", "--no-deps", "-T", "worker", "attachment-jobs") + listed = compose(*command, "failed") + check(any(line.startswith(job + "\t") for line in listed.splitlines()), "failed job was not listed") + check(f"QUEUED {job}" in compose(*command, "replay", job, "local-verification"), "replay was not queued") + check(metadata_state(queued["id"]) == "queued|1|t", "operator replay was not committed with outbox") + compose("start", "worker") + verify_result(client, reader, queued) + + refused = compose(*command, "replay", job, "local-verification", expected=2) + check("REJECTED" in refused, "successful job accepted a duplicate replay") + check(metadata_state(queued["id"]).startswith("ready|2|"), "successful result changed on duplicate replay") + print(f"PASS: failed job listed, replayed with same id on attempt 2, and replay refused after ready; id={queued['id']}", flush=True) + finally: + subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT) + compose("start", "s3", "worker") + + +def main(): + client = Client(timeout=15) + check(client.base.startswith(("http://localhost:", "http://127.0.0.1:")), "use a local Compose stand") + actor, reader, _, chat = client.bootstrap() + producer_crash(client, actor, reader, chat) + worker_crash(client, actor, reader, chat) + operator_replay(client, actor, reader, chat) + + +if __name__ == "__main__": + main() diff --git a/scripts/attachment-worker-recovery.py b/scripts/attachment-worker-recovery.py index 59559b6b..0a259886 100644 --- a/scripts/attachment-worker-recovery.py +++ b/scripts/attachment-worker-recovery.py @@ -54,7 +54,7 @@ def main(): thumbnail = client.call('GET', path + '/content?variant=thumbnail', actor=bob)['bytes'] check(downloaded == original, 'original changed') check(jpeg_size(thumbnail) == (256, 160), 'incorrect thumbnail dimensions') - variant = attachment['variants'][0] + variant = next(item for item in attachment['variants'] if item['name'] == 'thumbnail') check(variant['sha256'] == hashlib.sha256(thumbnail).hexdigest(), 'thumbnail checksum mismatch') client.call('GET', path, 404, actor=eve) print(f"PASS: same accepted job reached ready; original unchanged; thumbnail JPEG 256x160 with matching checksum; attachment_id={attachment['id']}", flush=True) diff --git a/scripts/check-s3-permissions.sh b/scripts/check-s3-permissions.sh index 416bdeb3..e0db025d 100755 --- a/scripts/check-s3-permissions.sh +++ b/scripts/check-s3-permissions.sh @@ -12,7 +12,7 @@ if mc anonymous set download "app/$S3_BUCKET" >/dev/null 2>&1; then exit 1; fi printf test > /tmp/test-object if mc cp /tmp/test-object "app/$S3_BUCKET/forbidden-prefix/object" >/dev/null 2>&1; then exit 1; fi mc stat "app/$S3_BUCKET" >/dev/null -printf "PASS: app cannot administer MinIO, create buckets, change policy or write outside originals/\n" +printf "PASS: app cannot administer MinIO, create buckets, change policy or write to an unrelated prefix\n" ' if docker compose exec -T lb wget -q -T 2 -O /dev/null http://s3:9001; then echo 'FAIL: console reachable from shared network' diff --git a/src/main/kotlin/com/mtusi/messenger/main/App.kt b/src/main/kotlin/com/mtusi/messenger/main/App.kt index 97e9e9a9..f769402f 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/App.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/App.kt @@ -1,5 +1,6 @@ package com.mtusi.messenger.main +import com.mtusi.messenger.main.operator.runAttachmentJobsCommand import com.mtusi.messenger.main.plugins.authentication.configureAuthentication import com.mtusi.messenger.main.plugins.configureAttachmentCleanup import com.mtusi.messenger.main.plugins.configureCallLogging @@ -29,6 +30,10 @@ fun main(args: Array) { return migrateDatabase() } + if (command == "attachment-jobs") { + return runAttachmentJobsCommand(args.drop(1)) + } + if (command == "worker") { val workerArgs = arrayOf( "-config=worker.conf", diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt index 5ebfb74e..02a5ba46 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt @@ -20,7 +20,9 @@ import kotlinx.coroutines.ensureActive import kotlinx.coroutines.withContext import org.koin.ktor.ext.inject -fun Routing.configureHealthRouting() = route("/health") { +fun Routing.configureHealthRouting( + requireBroker: Boolean = false +) = route("/health") { get("live") { call.respond(HttpStatusCode.OK, HealthResponse()) @@ -29,7 +31,7 @@ fun Routing.configureHealthRouting() = route("/health") { val storage by inject() val broker by inject() - // Readiness checks PostgreSQL, S3 and RabbitMQ. Redis has a DB fallback. + // API uploads persist in PostgreSQL and S3. Only the worker requires RabbitMQ. get("ready") { // SIGTERM received: leave the balancer's rotation while requests in flight are still being finished. @@ -69,7 +71,7 @@ fun Routing.configureHealthRouting() = route("/health") { return@get } - if (!broker.ready()) { + if (requireBroker && !broker.ready()) { call.respondError(ErrorCode.UNAVAILABLE, "job broker unavailable") return@get } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt index 2e64b26d..2e035ba0 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt @@ -13,12 +13,12 @@ object AttachmentJobModel : UUIDTable("attachment_jobs") { val pipelineVersion = integer("pipeline_version").default(1) val attempts = integer("attempts").default(0) + val replayBaseAttempt = integer("replay_base_attempt").default(0) val leaseToken = javaUUID("lease_token").nullable() val leaseUntil = timestamp("lease_until").nullable() val lastErrorCode = varchar("last_error_code", 64).nullable() val nextAttemptAt = timestamp("next_attempt_at").nullable() val createdAt = timestamp("created_at") val legacyOriginal = bool("legacy_original").default(false) - val legacyDispatchPending = bool("legacy_dispatch_pending").default(false) } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobReplayModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobReplayModel.kt new file mode 100644 index 00000000..09f2348f --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobReplayModel.kt @@ -0,0 +1,18 @@ +package com.mtusi.messenger.main.infrastructure.models.attachment + +import org.jetbrains.exposed.v1.core.dao.id.java.UUIDTable +import org.jetbrains.exposed.v1.datetime.timestamp + +object AttachmentJobReplayModel : UUIDTable("attachment_job_replays") { + + val jobId = reference("job_id", AttachmentJobModel) + val operator = varchar("operator", 80) + val previousErrorCode = varchar("previous_error_code", 64) + val attempt = integer("attempt") + val createdAt = timestamp("created_at") + + init { + uniqueIndex(jobId, attempt) + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt index 027f3b7e..e65e3cb2 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt @@ -16,5 +16,6 @@ object AttachmentModel : UUIDTable("attachments") { val updatedAt = timestamp("updated_at") val status = varchar("status", 16).default("ready") val stagingCleaned = bool("staging_cleaned").default(false) + val storageCleanupStarted = bool("storage_cleanup_started").default(false) val errorCode = varchar("error_code", 64).nullable() } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentOutboxModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentOutboxModel.kt new file mode 100644 index 00000000..13a91a25 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentOutboxModel.kt @@ -0,0 +1,25 @@ +package com.mtusi.messenger.main.infrastructure.models.attachment + +import org.jetbrains.exposed.v1.core.dao.id.java.UUIDTable +import org.jetbrains.exposed.v1.core.java.javaUUID +import org.jetbrains.exposed.v1.datetime.CurrentTimestamp +import org.jetbrains.exposed.v1.datetime.timestamp + +object AttachmentOutboxModel : UUIDTable("attachment_outbox") { + + val jobId = reference("job_id", AttachmentJobModel) + val attempt = integer("attempt") + val availableAt = timestamp("available_at").defaultExpression(CurrentTimestamp) + val leaseToken = javaUUID("lease_token").nullable() + val leaseUntil = timestamp("lease_until").nullable() + + init { + index( + customIndexName = "attachment_outbox_due_idx", + isUnique = false, + availableAt, + id + ) + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt index fd4086db..9b46beb9 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt @@ -23,10 +23,6 @@ interface AttachmentRepository { errorCode: String? = null ) - fun markEnqueueFailed( - id: UUID - ) - fun markStagingCleaned( id: UUID ) @@ -35,9 +31,14 @@ interface AttachmentRepository { before: Instant ): List + fun claimExpiredStorage( + attachment: ExpiredAttachmentStorage, + before: Instant + ): Boolean + fun publish( id: UUID, - variant: AttachmentVariant + variants: List ) fun attach( diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt index f1144526..df8c3b43 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt @@ -14,7 +14,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.less import org.jetbrains.exposed.v1.jdbc.insert -import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.batchUpsert import org.jetbrains.exposed.v1.jdbc.select import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update @@ -99,20 +99,6 @@ class ExposedAttachmentRepository : AttachmentRepository { } } - override fun markEnqueueFailed( - id: UUID - ) { - AttachmentModel.update( - where = { - (AttachmentModel.id eq id) and (AttachmentModel.status eq "queued") - } - ) { - it[status] = "failed" - it[errorCode] = "enqueue_failed" - it[updatedAt] = Clock.System.nowMicros() - } - } - override fun markStagingCleaned( id: UUID ) { @@ -145,16 +131,16 @@ class ExposedAttachmentRepository : AttachmentRepository { override fun publish( id: UUID, - variant: AttachmentVariant + variants: List ) { - AttachmentVariantModel.insertIgnore { - it[attachmentId] = id - it[name] = variant.name - it[contentType] = variant.contentType - it[sizeBytes] = variant.sizeBytes - it[width] = variant.width - it[height] = variant.height - it[sha256] = variant.sha256 + AttachmentVariantModel.batchUpsert(variants) { variant -> + this[AttachmentVariantModel.attachmentId] = id + this[AttachmentVariantModel.name] = variant.name + this[AttachmentVariantModel.contentType] = variant.contentType + this[AttachmentVariantModel.sizeBytes] = variant.sizeBytes + this[AttachmentVariantModel.width] = variant.width + this[AttachmentVariantModel.height] = variant.height + this[AttachmentVariantModel.sha256] = variant.sha256 } setState( @@ -163,6 +149,25 @@ class ExposedAttachmentRepository : AttachmentRepository { ) } + override fun claimExpiredStorage( + attachment: ExpiredAttachmentStorage, + before: Instant + ): Boolean { + + val status = if (attachment.failed) "failed" else "ready" + + return AttachmentModel.update( + where = { + (AttachmentModel.id eq attachment.id) + .and(AttachmentModel.status eq status) + .and(AttachmentModel.updatedAt less before) + .and(AttachmentModel.stagingCleaned eq false) + } + ) { + it[storageCleanupStarted] = true + } == 1 + } + override fun attach( messageId: UUID, ids: List diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt index 270d47e2..64ea2c2c 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt @@ -3,6 +3,8 @@ package com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant import java.util.UUID +import kotlin.time.Duration +import kotlin.time.Instant interface AttachmentJobRepository { @@ -14,13 +16,14 @@ interface AttachmentJobRepository { job: AttachmentJob, token: UUID, maxAttempts: Int, - leaseSeconds: Long + leaseSeconds: Long, + retryDelay: (Int) -> Duration = { Duration.ZERO } ): JobClaim fun complete( job: AttachmentJob, token: UUID, - variant: AttachmentVariant + variants: List ): Boolean fun fail( @@ -28,19 +31,39 @@ interface AttachmentJobRepository { token: UUID, errorCode: String, permanent: Boolean, - maxAttempts: Int + maxAttempts: Int, + retryDelay: Duration = Duration.ZERO ): Boolean - fun findPendingLegacyJobs( + fun findFailed( limit: Int - ): List + ): List - fun markLegacyJobDispatched( - id: UUID - ) + fun replay( + id: UUID, + operator: String + ): JobReplayResult } +data class FailedAttachmentJob( + val jobId: UUID, + val attachmentId: UUID, + val attempts: Int, + val errorCode: String, + val lastErrorCode: String?, + val failedAt: Instant, + val replayable: Boolean +) + +enum class JobReplayResult { + QUEUED, + NOT_FOUND, + NOT_FAILED, + PERMANENT_FAILURE, + SOURCE_EXPIRED +} + sealed interface JobClaim { data object Complete : JobClaim @@ -53,7 +76,8 @@ sealed interface JobClaim { data class Acquired( val attempt: Int, - val legacyOriginal: Boolean = false + val legacyOriginal: Boolean = false, + val attemptInRun: Int = attempt ) : JobClaim } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentOutboxRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentOutboxRepository.kt new file mode 100644 index 00000000..8da72c51 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentOutboxRepository.kt @@ -0,0 +1,33 @@ +package com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import java.util.UUID +import kotlin.time.Instant + +interface AttachmentOutboxRepository { + + fun enqueue( + job: AttachmentJob, + availableAt: Instant = job.createdAt + ) + + fun claimDue( + token: UUID + ): PendingAttachmentPublish? + + fun complete( + id: UUID, + token: UUID + ) + + fun reschedule( + id: UUID, + token: UUID + ) + +} + +data class PendingAttachmentPublish( + val id: UUID, + val job: AttachmentJob +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt index 7de5e212..91551a43 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt @@ -3,16 +3,12 @@ package com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobReplayModel import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository -import com.mtusi.messenger.main.utils.exposed.operations.plus import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder -import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq -import org.jetbrains.exposed.v1.core.greater -import org.jetbrains.exposed.v1.core.isNotNull -import org.jetbrains.exposed.v1.core.longLiteral import org.jetbrains.exposed.v1.datetime.CurrentTimestamp import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.select @@ -20,10 +16,14 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update import org.koin.core.annotation.Single import java.util.UUID +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds +import kotlin.time.Instant @Single class ExposedAttachmentJobRepository( - private val attachmentRepository: AttachmentRepository + private val attachmentRepository: AttachmentRepository, + private val outboxRepository: AttachmentOutboxRepository ) : AttachmentJobRepository { override fun create( @@ -41,67 +41,81 @@ class ExposedAttachmentJobRepository( job: AttachmentJob, token: UUID, maxAttempts: Int, - leaseSeconds: Long + leaseSeconds: Long, + retryDelay: (Int) -> Duration ): JobClaim { val row = lock(job) ?: return JobClaim.Complete - val status = AttachmentModel - .select(AttachmentModel.status) - .where(AttachmentModel.id eq job.attachmentId) - .forUpdate() - .single()[AttachmentModel.status] + val attachment = lockAttachment(job.attachmentId) + val status = attachment[AttachmentModel.status] if (status in setOf("ready", "failed")) { return JobClaim.Complete } - val activeLease = AttachmentJobModel.leaseUntil.isNotNull() - .and(AttachmentJobModel.leaseUntil greater CurrentTimestamp) - - val busy = AttachmentJobModel - .select(activeLease) - .where(AttachmentJobModel.id eq job.jobId) - .single() - .get(activeLease) == true - - if (busy) { - return JobClaim.Busy - } - val attempts = row[AttachmentJobModel.attempts] + val attemptsInRun = attempts - row[AttachmentJobModel.replayBaseAttempt] + val leaseUntil = row[AttachmentJobModel.leaseUntil] + val now = row[CurrentTimestamp] + val nextAttempt = attempts + 1 - if (attempts >= maxAttempts) { - attachmentRepository.setState( - id = job.attachmentId, - status = "failed", - errorCode = "retry_exhausted" + if (leaseUntil != null) { + // The delivery for the live claim is the only one that can schedule recovery. + if (job.attempt != attempts) { + return JobClaim.Complete + } + + if (leaseUntil > now) { + outboxRepository.enqueue( + job = job, + availableAt = leaseUntil + ) + + return JobClaim.Busy + } + + if (attemptsInRun >= maxAttempts) { + markFailed( + job = job, + cause = "lease_expired", + permanent = false + ) + + return JobClaim.Complete + } + + queueRetry( + job = job.copy(attempt = nextAttempt), + errorCode = "lease_expired", + availableAt = now + retryDelay(attemptsInRun) ) - clearLease(job.jobId) + return JobClaim.Retry(attempt = nextAttempt) + } + // Once an attempt moves on, its old deliveries cannot skip backoff or restart a replay. + val nextAttemptAt = row[AttachmentJobModel.nextAttemptAt] + + if (job.attempt != nextAttempt || (nextAttemptAt != null && nextAttemptAt > now)) { return JobClaim.Complete } - val nextAttempt = attempts + 1 + if (attemptsInRun >= maxAttempts) { + markFailed( + job = job, + cause = row[AttachmentJobModel.lastErrorCode] ?: "retry_exhausted", + permanent = false + ) - if (job.attempt != nextAttempt) { - if (row[AttachmentJobModel.leaseToken] != null) { - queueRetry( - id = job.jobId, - attachmentId = job.attachmentId, - errorCode = "lease_expired" - ) - } - - return JobClaim.Retry(attempt = nextAttempt) + return JobClaim.Complete } AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { it[this.attempts] = nextAttempt it[leaseToken] = token - it[leaseUntil] = CurrentTimestamp + longLiteral(leaseSeconds) - it[nextAttemptAt] = null + it[this.leaseUntil] = now + leaseSeconds.seconds + it[this.nextAttemptAt] = null } attachmentRepository.setState( @@ -111,14 +125,15 @@ class ExposedAttachmentJobRepository( return JobClaim.Acquired( attempt = nextAttempt, - legacyOriginal = row[AttachmentJobModel.legacyOriginal] + legacyOriginal = row[AttachmentJobModel.legacyOriginal], + attemptInRun = attemptsInRun + 1 ) } override fun complete( job: AttachmentJob, token: UUID, - variant: AttachmentVariant + variants: List ): Boolean { val row = lock(job) ?: return false @@ -129,7 +144,7 @@ class ExposedAttachmentJobRepository( attachmentRepository.publish( id = job.attachmentId, - variant = variant + variants = variants ) clearLease(job.jobId) @@ -142,7 +157,8 @@ class ExposedAttachmentJobRepository( token: UUID, errorCode: String, permanent: Boolean, - maxAttempts: Int + maxAttempts: Int, + retryDelay: Duration ): Boolean { val row = lock(job) ?: return true @@ -151,90 +167,181 @@ class ExposedAttachmentJobRepository( return false } - val terminal = permanent || row[AttachmentJobModel.attempts] >= maxAttempts + val attempts = row[AttachmentJobModel.attempts] + val attemptsInRun = attempts - row[AttachmentJobModel.replayBaseAttempt] + val terminal = permanent || attemptsInRun >= maxAttempts if (!terminal) { queueRetry( - id = job.jobId, - attachmentId = job.attachmentId, - errorCode = errorCode + job = job.copy(attempt = attempts + 1), + errorCode = errorCode, + availableAt = row[CurrentTimestamp] + retryDelay ) return false } + markFailed( + job = job, + cause = errorCode, + permanent = permanent + ) + + return true + } + + private fun queueRetry( + job: AttachmentJob, + errorCode: String, + availableAt: Instant + ) { attachmentRepository.setState( id = job.attachmentId, - status = "failed", - errorCode = if (permanent) errorCode else "retry_exhausted" + status = "queued" ) AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { it[leaseToken] = null it[leaseUntil] = null it[lastErrorCode] = errorCode - it[nextAttemptAt] = null + it[nextAttemptAt] = availableAt } - return true + outboxRepository.enqueue( + job = job, + availableAt = availableAt + ) } - private fun queueRetry( - id: UUID, - attachmentId: UUID, - errorCode: String + private fun markFailed( + job: AttachmentJob, + cause: String, + permanent: Boolean ) { + attachmentRepository.setState( + id = job.attachmentId, + status = "failed", + errorCode = if (permanent) cause else "retry_exhausted" + ) + + AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { + it[leaseToken] = null + it[leaseUntil] = null + it[lastErrorCode] = cause + it[nextAttemptAt] = null + } + } + + override fun findFailed( + limit: Int + ): List { + return AttachmentJobModel + .innerJoin(AttachmentModel) + .selectAll() + .where(AttachmentModel.status eq "failed") + .orderBy(AttachmentModel.updatedAt to SortOrder.DESC, AttachmentJobModel.id to SortOrder.ASC) + .limit(limit) + .map { row -> + FailedAttachmentJob( + jobId = row[AttachmentJobModel.id].value, + attachmentId = row[AttachmentJobModel.attachmentId].value, + attempts = row[AttachmentJobModel.attempts], + errorCode = checkNotNull(row[AttachmentModel.errorCode]), + lastErrorCode = row[AttachmentJobModel.lastErrorCode], + failedAt = row[AttachmentModel.updatedAt], + replayable = replayRestriction(row) == null + ) + } + } + + override fun replay( + id: UUID, + operator: String + ): JobReplayResult { + val row = lockById(id) ?: return JobReplayResult.NOT_FOUND + val attachmentId = row[AttachmentJobModel.attachmentId].value + val attachment = lockAttachment(attachmentId) + val restriction = replayRestriction(attachment) + + if (restriction != null) { + return restriction + } + + val attempts = row[AttachmentJobModel.attempts] + val now = row[CurrentTimestamp] + val job = AttachmentJob( + jobId = id, + attachmentId = attachmentId, + createdAt = row[AttachmentJobModel.createdAt], + pipelineVersion = row[AttachmentJobModel.pipelineVersion], + attempt = attempts + 1 + ) + + AttachmentJobModel.update(where = { AttachmentJobModel.id eq id }) { + it[replayBaseAttempt] = attempts + it[leaseToken] = null + it[leaseUntil] = null + it[lastErrorCode] = null + it[nextAttemptAt] = now + } + + AttachmentJobReplayModel.insert { + it[jobId] = id + it[this.operator] = operator + it[previousErrorCode] = checkNotNull(attachment[AttachmentModel.errorCode]) + it[attempt] = job.attempt + it[createdAt] = now + } + attachmentRepository.setState( id = attachmentId, status = "queued" ) - AttachmentJobModel.update(where = { AttachmentJobModel.id eq id }) { - it[leaseToken] = null - it[leaseUntil] = null - it[lastErrorCode] = errorCode - it[nextAttemptAt] = CurrentTimestamp - } + outboxRepository.enqueue(job, availableAt = now) + + return JobReplayResult.QUEUED } - override fun findPendingLegacyJobs( - limit: Int - ): List { - return AttachmentJobModel - .selectAll() - .where(AttachmentJobModel.legacyDispatchPending eq true) - .orderBy(AttachmentJobModel.createdAt to SortOrder.ASC, AttachmentJobModel.id to SortOrder.ASC) - .limit(limit) - .map { - AttachmentJob( - jobId = it[AttachmentJobModel.id].value, - attachmentId = it[AttachmentJobModel.attachmentId].value, - pipelineVersion = it[AttachmentJobModel.pipelineVersion], - createdAt = it[AttachmentJobModel.createdAt] - ) - } - } - - override fun markLegacyJobDispatched( - id: UUID - ) { - AttachmentJobModel.update(where = { AttachmentJobModel.id eq id }) { - it[legacyDispatchPending] = false + private fun replayRestriction( + attachment: ResultRow + ): JobReplayResult? { + return when { + attachment[AttachmentModel.status] != "failed" -> JobReplayResult.NOT_FAILED + attachment[AttachmentModel.errorCode] == "invalid_image" -> JobReplayResult.PERMANENT_FAILURE + attachment[AttachmentModel.stagingCleaned] || attachment[AttachmentModel.storageCleanupStarted] -> + JobReplayResult.SOURCE_EXPIRED + else -> null } } private fun lock( job: AttachmentJob + ): ResultRow? { + return lockById(job.jobId)?.takeIf { + it[AttachmentJobModel.attachmentId].value == job.attachmentId && + it[AttachmentJobModel.pipelineVersion] == job.pipelineVersion + } + } + + private fun lockById( + id: UUID ): ResultRow? { return AttachmentJobModel - .selectAll() - .where(AttachmentJobModel.id eq job.jobId) + .select(AttachmentJobModel.columns + CurrentTimestamp) + .where(AttachmentJobModel.id eq id) .forUpdate() .firstOrNull() - ?.takeIf { - it[AttachmentJobModel.attachmentId].value == job.attachmentId && - it[AttachmentJobModel.pipelineVersion] == job.pipelineVersion - } + } + + private fun lockAttachment( + id: UUID + ): ResultRow { + return AttachmentModel + .selectAll() + .where(AttachmentModel.id eq id) + .forUpdate() + .single() } private fun clearLease( diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentOutboxRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentOutboxRepository.kt new file mode 100644 index 00000000..983ffa4d --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentOutboxRepository.kt @@ -0,0 +1,110 @@ +package com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentOutboxModel +import com.mtusi.messenger.main.utils.exposed.operations.plus +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.isNull +import org.jetbrains.exposed.v1.core.lessEq +import org.jetbrains.exposed.v1.core.longLiteral +import org.jetbrains.exposed.v1.core.or +import org.jetbrains.exposed.v1.core.vendors.ForUpdateOption +import org.jetbrains.exposed.v1.datetime.CurrentTimestamp +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import org.koin.core.annotation.Single +import java.util.UUID +import kotlin.time.Instant + +private const val PUBLISH_LEASE_SECONDS = 30L +private const val PUBLISH_RETRY_SECONDS = 1L + +@Single +class ExposedAttachmentOutboxRepository : AttachmentOutboxRepository { + + override fun enqueue( + job: AttachmentJob, + availableAt: Instant + ) { + AttachmentOutboxModel.insert { + it[jobId] = job.jobId + it[attempt] = job.attempt + it[this.availableAt] = availableAt + } + } + + override fun claimDue( + token: UUID + ): PendingAttachmentPublish? { + + val row = AttachmentOutboxModel + .selectAll() + .where( + (AttachmentOutboxModel.availableAt lessEq CurrentTimestamp) and + (AttachmentOutboxModel.leaseUntil.isNull() or + (AttachmentOutboxModel.leaseUntil lessEq CurrentTimestamp)) + ) + .orderBy( + AttachmentOutboxModel.availableAt to SortOrder.ASC, + AttachmentOutboxModel.id to SortOrder.ASC + ) + .limit(1) + .forUpdate(ForUpdateOption.PostgreSQL.ForUpdate(ForUpdateOption.PostgreSQL.MODE.SKIP_LOCKED)) + .firstOrNull() + ?: return null + + val id = row[AttachmentOutboxModel.id].value + val jobId = row[AttachmentOutboxModel.jobId].value + + AttachmentOutboxModel.update(where = { AttachmentOutboxModel.id eq id }) { + it[leaseToken] = token + it[leaseUntil] = CurrentTimestamp + longLiteral(PUBLISH_LEASE_SECONDS) + } + + val job = AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.id eq jobId) + .single() + + return PendingAttachmentPublish( + id = id, + job = AttachmentJob( + jobId = jobId, + attachmentId = job[AttachmentJobModel.attachmentId].value, + attempt = row[AttachmentOutboxModel.attempt], + pipelineVersion = job[AttachmentJobModel.pipelineVersion], + createdAt = job[AttachmentJobModel.createdAt] + ) + ) + } + + override fun complete( + id: UUID, + token: UUID + ) { + AttachmentOutboxModel.deleteWhere { + (AttachmentOutboxModel.id eq id) and (leaseToken eq token) + } + } + + override fun reschedule( + id: UUID, + token: UUID + ) { + AttachmentOutboxModel.update( + where = { + (AttachmentOutboxModel.id eq id) and (AttachmentOutboxModel.leaseToken eq token) + } + ) { + it[availableAt] = CurrentTimestamp + longLiteral(PUBLISH_RETRY_SECONDS) + it[leaseToken] = null + it[leaseUntil] = null + } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt index d7b41db8..e110b7ce 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt @@ -37,19 +37,27 @@ class AttachmentCleanupService( } } - val expired = suspendedTransaction { - attachmentRepository.findExpiredStorage( - before = Clock.System.now() - config.stagingRetentionSeconds.seconds - ) - } + val before = Clock.System.now() - config.stagingRetentionSeconds.seconds + val expired = suspendedTransaction { attachmentRepository.findExpiredStorage(before) } - expired.forEach { attachment -> + for (attachment in expired) { try { + val claimed = suspendedTransaction { + attachmentRepository.claimExpiredStorage( + attachment = attachment, + before = before + ) + } + + if (!claimed) { + continue + } + when { attachment.failed && !attachment.legacyOriginal -> objectStorageService.delete(attachment.id) attachment.failed -> - objectStorageService.deleteThumbnail(attachment.id) + objectStorageService.deleteVariants(attachment.id) !attachment.legacyOriginal -> objectStorageService.deleteStaging(attachment.id) } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentImageProcessor.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentImageProcessor.kt new file mode 100644 index 00000000..c8b98182 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentImageProcessor.kt @@ -0,0 +1,137 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage +import com.mtusi.messenger.main.infrastructure.types.ApiException +import com.mtusi.messenger.main.infrastructure.types.ErrorCode +import kotlinx.coroutines.runBlocking +import java.io.DataInputStream +import java.io.DataOutputStream +import java.nio.file.Path +import kotlin.system.exitProcess +import kotlin.time.Duration.Companion.seconds + +private const val INVALID_IMAGE = 64 +private const val IMAGE_LIMIT = 65 +private const val ENCODER_TIMEOUT = 66 +private const val JVM_OUT_OF_MEMORY = 3 +private val OUTPUT_TYPES = linkedMapOf( + "thumbnail" to "image/jpeg", + "crop" to "image/jpeg", + "optimized" to "image/webp" +) + +internal object AttachmentImageProcessor { + + suspend fun process( + bytes: ByteArray, + declaredType: String + ): Map { + + if (bytes.size > MAX_ATTACHMENT_BYTES) { + throw ApiException( + code = ErrorCode.PAYLOAD_TOO_LARGE, + message = "file exceeds 10 MiB" + ) + } + + val java = Path.of(System.getProperty("java.home"), "bin", "java") + val classpath = System.getProperty("messenger.image.classpath") + ?: System.getProperty("java.class.path") + + val response = runImageProcess( + command = listOf( + java.toString(), + "-Xmx384m", + "-XX:+UseSerialGC", + "-XX:ActiveProcessorCount=1", + "-XX:+ExitOnOutOfMemoryError", + "-Djava.awt.headless=true", + "-cp", classpath, + AttachmentImageProcessor::class.java.name, + declaredType + ), + input = bytes, + outputLimit = MAX_ATTACHMENT_BYTES + 1024 * 1024, + timeout = 25.seconds + ) + + when (response.exitCode) { + INVALID_IMAGE -> throw ApiException( + code = ErrorCode.UNSUPPORTED_MEDIA_TYPE, + message = "invalid source image" + ) + IMAGE_LIMIT, JVM_OUT_OF_MEMORY -> throw ApiException( + code = ErrorCode.PAYLOAD_TOO_LARGE, + message = "image exceeds processing limits" + ) + ENCODER_TIMEOUT -> throw ImageProcessTimeoutException() + else -> check(response.exitCode == 0) { "image processor failed" } + } + + return readVariants(response.output) + } + + @JvmStatic + fun main( + args: Array + ) { + + val declaredType = args.single() + val bytes = System.`in`.readNBytes(MAX_ATTACHMENT_BYTES + 1) + + val variants = try { + runBlocking { + DefaultImageProcessingService().generateVariants( + bytes = bytes, + declaredType = declaredType + ) + } + } catch (cause: ApiException) { + when (cause.code) { + ErrorCode.UNSUPPORTED_MEDIA_TYPE -> exitProcess(INVALID_IMAGE) + ErrorCode.PAYLOAD_TOO_LARGE -> exitProcess(IMAGE_LIMIT) + else -> throw cause + } + } catch (_: ImageProcessTimeoutException) { + exitProcess(ENCODER_TIMEOUT) + } + + DataOutputStream(System.out).use { output -> + for (name in OUTPUT_TYPES.keys) { + val variant = variants.getValue(name) + output.writeInt(variant.width) + output.writeInt(variant.height) + output.writeInt(variant.bytes.size) + output.write(variant.bytes) + } + } + } + + private fun readVariants( + bytes: ByteArray + ): Map = DataInputStream(bytes.inputStream()).use { input -> + + val variants = OUTPUT_TYPES.mapValues { (_, contentType) -> + val width = input.readInt() + val height = input.readInt() + val size = input.readInt() + + check(width in 1..1280 && height in 1..1280 && size in 1..MAX_ATTACHMENT_BYTES) + + val image = ByteArray(size) + input.readFully(image) + + ProcessedImage( + bytes = image, + width = width, + height = height, + contentType = contentType + ) + } + + check(input.read() == -1) { "unexpected image process output" } + + variants + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt index 29812a17..e4a8ef85 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt @@ -6,17 +6,18 @@ import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.JobClaim -import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ApiException import com.mtusi.messenger.main.infrastructure.types.ErrorCode import com.mtusi.messenger.main.plugins.suspendedTransaction import com.mtusi.messenger.main.utils.computeSha256 import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.delay import kotlinx.coroutines.withTimeoutOrNull import org.koin.core.annotation.Single import java.util.UUID +import kotlin.random.Random +import kotlin.time.Duration +import kotlin.system.exitProcess import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds @@ -26,7 +27,6 @@ class DefaultAttachmentProcessingService( private val attachmentRepository: AttachmentRepository, private val objectStorageService: ObjectStorageService, private val imageProcessingService: ImageProcessingService, - private val jobBrokerService: JobBrokerService, private val config: WorkerConfig ) { @@ -40,28 +40,13 @@ class DefaultAttachmentProcessingService( job = job, token = token, maxAttempts = config.maxAttempts, - leaseSeconds = config.timeoutSeconds + 15 + leaseSeconds = config.timeoutSeconds + 15, + retryDelay = ::retryDelay ) } when (claim) { - JobClaim.Complete -> return true - - JobClaim.Busy -> { - // Pace redeliveries while the lease is active. Publish to the tail and - // confirm before acknowledging the original delivery. - delay(250.milliseconds) - jobBrokerService.publish(job) - - return true - } - - is JobClaim.Retry -> { - jobBrokerService.publish(job.copy(attempt = claim.attempt)) - - return true - } - + JobClaim.Complete, JobClaim.Busy, is JobClaim.Retry -> return true is JobClaim.Acquired -> Unit } @@ -86,31 +71,37 @@ class DefaultAttachmentProcessingService( val errorCode = when { isPermanent -> "invalid_image" - cause is ProcessingTimeoutException -> "processing_timeout" + cause is ProcessingTimeoutException || cause is ImageProcessTimeoutException -> "processing_timeout" cause is ApiException && cause.code == ErrorCode.UNAVAILABLE -> "storage_unavailable" else -> "processing_failed" } - val isTerminal = suspendedTransaction { + suspendedTransaction { jobRepository.fail( job = job, token = token, errorCode = errorCode, permanent = isPermanent, - maxAttempts = config.maxAttempts + maxAttempts = config.maxAttempts, + retryDelay = retryDelay(claim.attemptInRun) ) } - if (!isTerminal) { - jobBrokerService.publish(job.copy(attempt = claim.attempt + 1)) - } - return true } } private class ProcessingTimeoutException : RuntimeException() + private fun retryDelay( + attempts: Int + ): Duration { + val exponent = (attempts - 1).coerceIn(0, 5) + val upperMillis = (1000L shl exponent).coerceAtMost(30_000L) + + return Random.nextLong(upperMillis / 2, upperMillis + 1).milliseconds + } + private suspend fun processAttachment( job: AttachmentJob, token: UUID, @@ -134,11 +125,15 @@ class DefaultAttachmentProcessingService( ) } - val thumbnail = imageProcessingService.thumbnail( + val images = imageProcessingService.variants( bytes = bytes, declaredType = attachment.contentType ) + check(images.keys == setOf("thumbnail", "crop", "optimized")) { + "Image processor did not produce all required variants" + } + if (!legacyOriginal) { objectStorageService.put( id = attachment.id, @@ -147,23 +142,33 @@ class DefaultAttachmentProcessingService( ) } - objectStorageService.putThumbnail( - id = attachment.id, - bytes = thumbnail.bytes - ) + val variants = images.map { (name, image) -> + objectStorageService.putVariant( + id = attachment.id, + name = name, + bytes = image.bytes + ) + + AttachmentVariant( + name = name, + contentType = image.contentType, + sizeBytes = image.bytes.size.toLong(), + width = image.width, + height = image.height, + sha256 = image.bytes.computeSha256() + ) + } + + // Enabled only on a disposable worker used by the recovery check. + if (System.getenv("LAB5_TEST_CRASH_AFTER_OBJECTS") == attachment.id.toString()) { + exitProcess(86) + } return suspendedTransaction { jobRepository.complete( job = job, token = token, - variant = AttachmentVariant( - name = "thumbnail", - contentType = thumbnail.contentType, - sizeBytes = thumbnail.bytes.size.toLong(), - width = thumbnail.width, - height = thumbnail.height, - sha256 = thumbnail.bytes.computeSha256() - ) + variants = variants ) } } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt index 8782680c..181d0229 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt @@ -5,17 +5,15 @@ import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.UploadJournalRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentOutboxRepository import com.mtusi.messenger.main.infrastructure.repositories.chat.ChatRepository -import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ApiException import com.mtusi.messenger.main.infrastructure.types.ErrorCode import com.mtusi.messenger.main.plugins.suspendedTransaction import com.mtusi.messenger.main.utils.computeSha256 import com.mtusi.messenger.main.utils.nowMicros -import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.withContext import org.koin.core.annotation.Single import java.io.File @@ -29,7 +27,7 @@ class DefaultAttachmentService( private val chatRepository: ChatRepository, private val objectStorageService: ObjectStorageService, private val jobRepository: AttachmentJobRepository, - private val jobBrokerService: JobBrokerService + private val outboxRepository: AttachmentOutboxRepository ) : AttachmentService { override suspend fun requireMember( @@ -97,37 +95,18 @@ class DefaultAttachmentService( createdAt = attachment.createdAt ) - try { - suspendedTransaction { - if (!journalRepository.lockForCompletion(attachment.id)) { - throw ApiException( - code = ErrorCode.UNAVAILABLE, - message = "upload expired" - ) - } - - attachmentRepository.create(attachment) - jobRepository.create(job) - journalRepository.complete(attachment.id) + suspendedTransaction { + if (!journalRepository.lockForCompletion(attachment.id)) { + throw ApiException( + code = ErrorCode.UNAVAILABLE, + message = "upload expired" + ) } - jobBrokerService.publish(job) - } catch (cause: CancellationException) { - withContext(NonCancellable) { - runCatching { - suspendedTransaction { - attachmentRepository.markEnqueueFailed(attachment.id) - } - } - } - - throw cause - } catch (cause: Exception) { - suspendedTransaction { - attachmentRepository.markEnqueueFailed(attachment.id) - } - - throw cause + attachmentRepository.create(attachment) + jobRepository.create(job) + outboxRepository.enqueue(job) + journalRepository.complete(attachment.id) } return attachment @@ -165,9 +144,12 @@ class DefaultAttachmentService( ) } - when (variant) { - "original" -> objectStorageService.get(attachment.id) - "thumbnail" -> objectStorageService.getThumbnail(attachment.id) + when { + variant == "original" -> objectStorageService.get(attachment.id) + attachment.variants.any { it.name == variant } -> objectStorageService.getVariant( + id = attachment.id, + name = variant + ) else -> throw ApiException( code = ErrorCode.NOT_FOUND, message = "variant not found" diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt index 8dd1a869..9ab66acc 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt @@ -9,6 +9,8 @@ import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import org.koin.core.annotation.Single import java.awt.Color @@ -18,20 +20,212 @@ import java.awt.image.BufferedImage import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.nio.ByteBuffer +import javax.imageio.IIOImage import javax.imageio.ImageIO +import javax.imageio.ImageWriteParam +import javax.imageio.stream.MemoryCacheImageOutputStream +import kotlin.time.Duration.Companion.seconds private const val THUMBNAIL_SIZE = 256 +private const val CROP_SIZE = 128 +private const val OPTIMIZED_SIZE = 1280 +private const val JPEG_QUALITY = 0.85f @Single class DefaultImageProcessingService : ImageProcessingService { - override suspend fun thumbnail( + private val decoder = Semaphore(1) + + override suspend fun variants( bytes: ByteArray, declaredType: String - ): ProcessedImage = withContext(Dispatchers.IO) { + ): Map = decoder.withPermit { + AttachmentImageProcessor.process( + bytes = bytes, + declaredType = declaredType + ) + } + + internal suspend fun thumbnail( + bytes: ByteArray, + declaredType: String + ): ProcessedImage = withSource(bytes, declaredType, ::thumbnail) + + internal suspend fun generateVariants( + bytes: ByteArray, + declaredType: String + ): Map = withSource(bytes, declaredType) { source -> + linkedMapOf( + "thumbnail" to thumbnail(source), + "crop" to crop(source), + "optimized" to optimized(source) + ) + } + + private suspend fun thumbnail( + source: SourceImage + ): ProcessedImage { + val (width, height) = source.fit(THUMBNAIL_SIZE) + + return render( + source = source, + width = width, + height = height, + transform = source.scaleTo(width, height), + contentType = "image/jpeg", + encode = ::encodeJpeg + ) + } + + private suspend fun crop( + source: SourceImage + ): ProcessedImage { + + val scale = CROP_SIZE.toDouble() / minOf(source.width, source.height) + val transform = AffineTransform.getTranslateInstance( + (CROP_SIZE - source.width * scale) / 2, + (CROP_SIZE - source.height * scale) / 2 + ).apply { + scale(scale, scale) + } + + return render( + source = source, + width = CROP_SIZE, + height = CROP_SIZE, + transform = transform, + contentType = "image/jpeg", + encode = ::encodeJpeg + ) + } + + private suspend fun optimized( + source: SourceImage + ): ProcessedImage { + val (width, height) = source.fit(OPTIMIZED_SIZE) + + return render( + source = source, + width = width, + height = height, + transform = source.scaleTo(width, height), + contentType = "image/webp", + encode = ::encodeWebp + ) + } + + private suspend fun render( + source: SourceImage, + width: Int, + height: Int, + transform: AffineTransform, + contentType: String, + encode: suspend (BufferedImage) -> ByteArray + ): ProcessedImage { currentCoroutineContext().ensureActive() + val preserveAlpha = contentType == "image/webp" + val imageType = if (preserveAlpha) BufferedImage.TYPE_INT_ARGB else BufferedImage.TYPE_INT_RGB + val rendered = BufferedImage(width, height, imageType) + + try { + val graphics = rendered.createGraphics() + + try { + if (!preserveAlpha) { + graphics.color = Color.WHITE + graphics.fillRect(0, 0, width, height) + } + + graphics.setRenderingHint( + RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BILINEAR + ) + graphics.setRenderingHint( + RenderingHints.KEY_RENDERING, + RenderingHints.VALUE_RENDER_QUALITY + ) + + transform.concatenate(source.orientation) + graphics.drawImage(source.original, transform, null) + } finally { + graphics.dispose() + } + + return ProcessedImage( + bytes = encode(rendered), + width = width, + height = height, + contentType = contentType + ) + } finally { + rendered.flush() + } + } + + private fun encodeJpeg( + image: BufferedImage + ): ByteArray { + + val output = ByteArrayOutputStream() + val writer = ImageIO.getImageWritersByFormatName("jpeg").next() + + try { + MemoryCacheImageOutputStream(output).use { stream -> + writer.output = stream + + val parameters = writer.defaultWriteParam.apply { + compressionMode = ImageWriteParam.MODE_EXPLICIT + compressionQuality = JPEG_QUALITY + } + + writer.write(null, IIOImage(image, null, null), parameters) + } + } finally { + writer.dispose() + } + + return output.toByteArray() + } + + private suspend fun encodeWebp( + image: BufferedImage + ): ByteArray { + + val png = ByteArrayOutputStream() + + MemoryCacheImageOutputStream(png).use { output -> + check(ImageIO.write(image, "png", output)) { "PNG encoder is unavailable" } + } + + val encoded = runImageProcess( + command = listOf( + "cwebp", + "-quiet", + "-q", "82", + "-m", "4", + "-alpha_q", "100", + "-metadata", "none", + "-o", "-", + "--", "-" + ), + input = png.toByteArray(), + outputLimit = MAX_ATTACHMENT_BYTES, + timeout = 15.seconds + ) + + check(encoded.exitCode == 0 && encoded.output.isNotEmpty()) { "WebP encode failed" } + + return encoded.output + } + + private suspend fun withSource( + bytes: ByteArray, + declaredType: String, + block: suspend (SourceImage) -> R + ): R = withContext(Dispatchers.IO) { + val original = ImageValidator.decode( bytes = bytes, declaredType = declaredType @@ -41,94 +235,57 @@ class DefaultImageProcessingService : ImageProcessingService { currentCoroutineContext().ensureActive() val orientation = readOrientation(bytes, declaredType) - val swapDimensions = orientation in 5..8 - val orientedWidth = if (swapDimensions) original.height else original.width - val orientedHeight = if (swapDimensions) original.width else original.height - val longestSide = maxOf(orientedWidth, orientedHeight) - val width = if (longestSide <= THUMBNAIL_SIZE) { - orientedWidth - } else { - maxOf(1, (orientedWidth.toLong() * THUMBNAIL_SIZE / longestSide).toInt()) - } - - val height = if (longestSide <= THUMBNAIL_SIZE) { - orientedHeight - } else { - maxOf(1, (orientedHeight.toLong() * THUMBNAIL_SIZE / longestSide).toInt()) - } - - val thumbnail = BufferedImage(width, height, BufferedImage.TYPE_INT_RGB) - - try { - val transform = AffineTransform.getScaleInstance( - width.toDouble() / orientedWidth, - height.toDouble() / orientedHeight - ).apply { - val orientationCorrection = orientationTransform( + block( + SourceImage( + original = original, + width = if (orientation in 5..8) original.height else original.width, + height = if (orientation in 5..8) original.width else original.height, + orientation = orientationTransform( orientation = orientation, width = original.width, height = original.height ) - - concatenate(orientationCorrection) - } - - drawThumbnail( - original = original, - thumbnail = thumbnail, - transform = transform ) - - currentCoroutineContext().ensureActive() - - val encoded = ByteArrayOutputStream().use { output -> - check(ImageIO.write(thumbnail, "jpeg", output)) { - "JPEG encoder is unavailable" - } - - output.toByteArray() - } - - ProcessedImage( - bytes = encoded, - width = width, - height = height - ) - } finally { - thumbnail.flush() - } + ) } finally { original.flush() } } - private fun drawThumbnail( - original: BufferedImage, - thumbnail: BufferedImage, - transform: AffineTransform + private class SourceImage( + val original: BufferedImage, + val width: Int, + val height: Int, + val orientation: AffineTransform ) { - val graphics = thumbnail.createGraphics() + fun fit( + size: Int + ): Pair { - try { - graphics.color = Color.WHITE - graphics.fillRect(0, 0, thumbnail.width, thumbnail.height) + val longest = maxOf(width, height) - graphics.setRenderingHint( - RenderingHints.KEY_INTERPOLATION, - RenderingHints.VALUE_INTERPOLATION_BILINEAR - ) + if (longest <= size) { + return width to height + } - graphics.setRenderingHint( - RenderingHints.KEY_RENDERING, - RenderingHints.VALUE_RENDER_QUALITY - ) + val scaledWidth = maxOf(1, (width.toLong() * size / longest).toInt()) + val scaledHeight = maxOf(1, (height.toLong() * size / longest).toInt()) - graphics.drawImage(original, transform, null) - } finally { - graphics.dispose() + return scaledWidth to scaledHeight } + + fun scaleTo( + width: Int, + height: Int + ): AffineTransform { + return AffineTransform.getScaleInstance( + width.toDouble() / this.width, + height.toDouble() / this.height + ) + } + } private fun readOrientation( diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcess.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcess.kt new file mode 100644 index 00000000..5ab8178b --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcess.kt @@ -0,0 +1,64 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.future.await +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import java.util.concurrent.TimeUnit +import kotlin.time.Duration + +internal class ImageProcessTimeoutException : RuntimeException("image processing deadline exceeded") + +internal class ImageProcessResult( + val exitCode: Int, + val output: ByteArray +) + +internal suspend fun runImageProcess( + command: List, + input: ByteArray, + outputLimit: Int, + timeout: Duration +): ImageProcessResult = withContext(Dispatchers.IO) { + + val process = ProcessBuilder(command) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start() + + coroutineScope { + val writer = async { + process.outputStream.use { it.write(input) } + } + + val reader = async { + process.inputStream.use { it.readNBytes(outputLimit + 1) } + } + + try { + withTimeoutOrNull(timeout) { + val exitCode = process.onExit().await().exitValue() + val output = reader.await() + writer.await() + + check(output.size <= outputLimit) { "image process output exceeded limit" } + + ImageProcessResult( + exitCode = exitCode, + output = output + ) + } ?: throw ImageProcessTimeoutException() + } finally { + // Kill before the scope joins children that may be waiting on the pipes. + if (process.isAlive) { + process.descendants().use { children -> + children.forEach { it.destroyForcibly() } + } + + process.destroyForcibly() + process.waitFor(1, TimeUnit.SECONDS) + } + } + } +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt index 581cc459..a7fd2487 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt @@ -4,9 +4,9 @@ import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage interface ImageProcessingService { - suspend fun thumbnail( + suspend fun variants( bytes: ByteArray, declaredType: String - ): ProcessedImage + ): Map } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/AttachmentOutboxDispatchService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/AttachmentOutboxDispatchService.kt new file mode 100644 index 00000000..37a69388 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/AttachmentOutboxDispatchService.kt @@ -0,0 +1,49 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentOutboxRepository +import com.mtusi.messenger.main.plugins.suspendedTransaction +import kotlinx.coroutines.CancellationException +import org.koin.core.annotation.Single +import java.util.UUID + +private const val PUBLISH_BATCH_SIZE = 32 + +@Single +class AttachmentOutboxDispatchService( + private val outboxRepository: AttachmentOutboxRepository, + private val jobBrokerService: JobBrokerService +) { + + suspend fun publishPending() { + + repeat(PUBLISH_BATCH_SIZE) { + val token = UUID.randomUUID() + val pending = suspendedTransaction { + outboxRepository.claimDue(token) + } ?: return + + try { + jobBrokerService.publish(pending.job) + } catch (cause: CancellationException) { + throw cause + } catch (cause: Exception) { + suspendedTransaction { + outboxRepository.reschedule( + id = pending.id, + token = token + ) + } + + throw cause + } + + suspendedTransaction { + outboxRepository.complete( + id = pending.id, + token = token + ) + } + } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt deleted file mode 100644 index 2027d4b4..00000000 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt +++ /dev/null @@ -1,28 +0,0 @@ -package com.mtusi.messenger.main.infrastructure.services.attachment.jobs - -import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository -import com.mtusi.messenger.main.plugins.suspendedTransaction -import org.koin.core.annotation.Single - -@Single -class LegacyAttachmentDispatchService( - private val jobRepository: AttachmentJobRepository, - private val jobBrokerService: JobBrokerService -) { - - suspend fun publishPending() { - - val jobs = suspendedTransaction { - jobRepository.findPendingLegacyJobs(limit = 32) - } - - for (job in jobs) { - jobBrokerService.publish(job) - - suspendedTransaction { - jobRepository.markLegacyJobDispatched(job.jobId) - } - } - } - -} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt index 2246ffc6..b87f3e67 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt @@ -25,16 +25,18 @@ interface ObjectStorageService { id: UUID ) - suspend fun putThumbnail( + suspend fun putVariant( id: UUID, + name: String, bytes: ByteArray ) - suspend fun getThumbnail( - id: UUID + suspend fun getVariant( + id: UUID, + name: String ): ByteArray - suspend fun deleteThumbnail( + suspend fun deleteVariants( id: UUID ) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt index 63e3a0fd..e7d82505 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt @@ -85,29 +85,33 @@ class S3ObjectStorageService( deleteKey(getStagingKey(id)) } - override suspend fun putThumbnail( + override suspend fun putVariant( id: UUID, + name: String, bytes: ByteArray ) { request { client.putObject { bucket = config.bucket - key = getThumbnailKey(id) + key = getVariantKey(id, name) body = bytes.asByteStream() - contentType = "image/jpeg" + contentType = if (name == "optimized") "image/webp" else "image/jpeg" contentLength = bytes.size.toLong() } } } - override suspend fun getThumbnail( - id: UUID - ): ByteArray = getBytes(getThumbnailKey(id)) + override suspend fun getVariant( + id: UUID, + name: String + ): ByteArray = getBytes(getVariantKey(id, name)) - override suspend fun deleteThumbnail( + override suspend fun deleteVariants( id: UUID ) { - deleteKey(getThumbnailKey(id)) + for (name in listOf("thumbnail", "crop", "optimized")) { + deleteKey(getVariantKey(id, name)) + } } override suspend fun delete( @@ -115,7 +119,7 @@ class S3ObjectStorageService( ) { deleteKey(getObjectKey(id)) deleteStaging(id) - deleteThumbnail(id) + deleteVariants(id) } override suspend fun ready(): Boolean { @@ -173,9 +177,19 @@ class S3ObjectStorageService( id: UUID ) = "staging/$id" - private fun getThumbnailKey( - id: UUID - ) = "variants/$id/v1/thumbnail.jpg" + private fun getVariantKey( + id: UUID, + name: String + ): String { + val filename = when (name) { + "thumbnail" -> "thumbnail.jpg" + "crop" -> "crop.jpg" + "optimized" -> "optimized.webp" + else -> error("Unknown attachment variant") + } + + return "variants/$id/v1/$filename" + } private suspend fun request(block: suspend () -> T): T { return try { diff --git a/src/main/kotlin/com/mtusi/messenger/main/operator/AttachmentJobsCommand.kt b/src/main/kotlin/com/mtusi/messenger/main/operator/AttachmentJobsCommand.kt new file mode 100644 index 00000000..32811a64 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/operator/AttachmentJobsCommand.kt @@ -0,0 +1,105 @@ +package com.mtusi.messenger.main.operator + +import com.mtusi.messenger.main.config.database.SerialDatabaseConfig +import com.mtusi.messenger.main.infrastructure.repositories.attachment.ExposedAttachmentRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.ExposedAttachmentJobRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.ExposedAttachmentOutboxRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.JobReplayResult +import com.mtusi.messenger.main.plugins.createDataSource +import com.mtusi.messenger.main.plugins.suspendedTransaction +import com.mtusi.messenger.main.plugins.validateDatabaseMigrations +import com.typesafe.config.ConfigFactory +import io.ktor.server.config.HoconApplicationConfig +import io.ktor.server.config.getAs +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.core.DatabaseConfig +import org.jetbrains.exposed.v1.jdbc.Database +import java.util.UUID +import kotlin.system.exitProcess + +private val operatorPattern = Regex("[A-Za-z0-9_.@-]{1,80}") + +fun runAttachmentJobsCommand( + args: List +) { + + val subcommand = args.firstOrNull() + val replayJobId = if (subcommand == "replay" && args.size == 3) { + runCatching { UUID.fromString(args[1]) } + .getOrNull() + ?.takeIf { it.toString().equals(args[1], ignoreCase = true) } + } else { + null + } + val isList = args == listOf("failed") + val isReplay = replayJobId != null && operatorPattern.matches(args[2]) + + if (!isList && !isReplay) { + System.err.println("Usage: attachment-jobs failed | attachment-jobs replay JOB_ID OPERATOR") + exitProcess(2) + } + + val config = HoconApplicationConfig(ConfigFactory.load()) + .property("database") + .getAs() + + val exitCode = config.createDataSource().use { dataSource -> + validateDatabaseMigrations(dataSource) + Database.connect( + datasource = dataSource, + databaseConfig = DatabaseConfig { + defaultMaxAttempts = 1 + } + ) + + val jobs = ExposedAttachmentJobRepository( + attachmentRepository = ExposedAttachmentRepository(), + outboxRepository = ExposedAttachmentOutboxRepository() + ) + + runBlocking { + if (isList) { + val failed = suspendedTransaction { + jobs.findFailed(limit = 100) + } + + println("job_id\tattachment_id\tattempts\terror_code\tlast_error_code\tfailed_at\treplayable") + + for (job in failed) { + println( + listOf( + job.jobId, + job.attachmentId, + job.attempts, + job.errorCode, + job.lastErrorCode.orEmpty(), + job.failedAt, + job.replayable + ).joinToString("\t") + ) + } + 0 + } else { + val id = checkNotNull(replayJobId) + val result = suspendedTransaction { + jobs.replay( + id = id, + operator = args[2] + ) + } + + if (result == JobReplayResult.QUEUED) { + println("QUEUED ${id}") + 0 + } else { + System.err.println("REJECTED ${result.name}") + 2 + } + } + } + } + + if (exitCode != 0) { + exitProcess(exitCode) + } +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt index d2c94b20..702aedee 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt @@ -2,8 +2,8 @@ package com.mtusi.messenger.main.plugins import com.mtusi.messenger.main.config.worker.WorkerConfig import com.mtusi.messenger.main.infrastructure.services.attachment.DefaultAttachmentProcessingService +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.AttachmentOutboxDispatchService import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService -import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.LegacyAttachmentDispatchService import io.ktor.server.application.Application import io.ktor.server.application.ApplicationStopping import io.ktor.server.application.log @@ -18,7 +18,7 @@ fun Application.configureAttachmentWorker() { val broker = get() val processingService = get() - val legacyDispatchService = get() + val dispatchService = get() val config = get() val workers = List(config.concurrency) { @@ -27,14 +27,14 @@ fun Application.configureAttachmentWorker() { } } - val legacyDispatch = launch { + val dispatch = launch { while (isActive) { try { - legacyDispatchService.publishPending() + dispatchService.publishPending() } catch (cause: CancellationException) { throw cause } catch (cause: Exception) { - log.warn("Legacy attachment dispatch failed ({}); retrying", cause.javaClass.simpleName) + log.warn("Attachment outbox dispatch failed ({}); retrying", cause.javaClass.simpleName) } delay(1.seconds) @@ -43,7 +43,7 @@ fun Application.configureAttachmentWorker() { monitor.subscribe(ApplicationStopping) { workers.forEach { it.cancel() } - legacyDispatch.cancel() + dispatch.cancel() broker.close() } diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt index 9a55309e..b2b3b0d1 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt @@ -3,7 +3,9 @@ package com.mtusi.messenger.main.plugins import com.mtusi.messenger.main.config.database.DatabaseConfig import com.mtusi.messenger.main.config.database.SerialDatabaseConfig import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobReplayModel import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentOutboxModel import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentVariantModel import com.mtusi.messenger.main.infrastructure.models.attachment.MessageAttachmentModel import com.mtusi.messenger.main.infrastructure.models.attachment.UploadJournalModel @@ -65,7 +67,7 @@ fun Application.configureDatabase() { } -private fun DatabaseConfig.createDataSource(): HikariDataSource { +internal fun DatabaseConfig.createDataSource(): HikariDataSource { return HikariDataSource( HikariConfig().apply { poolName = "messenger" @@ -126,6 +128,8 @@ private fun generateMigration(description: String) { AttachmentModel, AttachmentVariantModel, AttachmentJobModel, + AttachmentJobReplayModel, + AttachmentOutboxModel, UploadJournalModel, MessageAttachmentModel, UserModel, diff --git a/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt b/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt index a326856a..e448c301 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt @@ -32,7 +32,7 @@ fun Application.workerModule() { configureAttachmentCleanup() routing { - configureHealthRouting() + configureHealthRouting(requireBroker = true) } } diff --git a/src/main/resources/migrations/V10__attachment_job_replay.sql b/src/main/resources/migrations/V10__attachment_job_replay.sql new file mode 100644 index 00000000..4cfb918f --- /dev/null +++ b/src/main/resources/migrations/V10__attachment_job_replay.sql @@ -0,0 +1,13 @@ +ALTER TABLE attachment_jobs + ADD COLUMN replay_base_attempt INTEGER NOT NULL DEFAULT 0 + CHECK (replay_base_attempt >= 0 AND replay_base_attempt <= attempts); + +CREATE TABLE attachment_job_replays ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + operator VARCHAR(80) NOT NULL, + previous_error_code VARCHAR(64) NOT NULL, + attempt INTEGER NOT NULL CHECK (attempt > 0), + created_at TIMESTAMPTZ NOT NULL, + UNIQUE (job_id, attempt) +); diff --git a/src/main/resources/migrations/V11__complete_attachment_variants.sql b/src/main/resources/migrations/V11__complete_attachment_variants.sql new file mode 100644 index 00000000..5e68fc6a --- /dev/null +++ b/src/main/resources/migrations/V11__complete_attachment_variants.sql @@ -0,0 +1,36 @@ +ALTER TABLE attachments ADD COLUMN storage_cleanup_started BOOLEAN NOT NULL DEFAULT FALSE; + +-- Reuse existing images when upgrading a level-three deployment. +WITH upgraded_jobs AS ( + UPDATE attachment_jobs j + SET legacy_original = TRUE, + replay_base_attempt = j.attempts, + last_error_code = NULL, + next_attempt_at = CURRENT_TIMESTAMP + FROM attachments a + WHERE j.attachment_id = a.id + AND a.status = 'ready' + AND ( + SELECT COUNT(*) + FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.name IN ('thumbnail', 'crop', 'optimized') + ) < 3 + RETURNING j.id, j.attachment_id, j.attempts +), queued_attachments AS ( + UPDATE attachments a + SET status = 'queued', + error_code = NULL, + updated_at = CURRENT_TIMESTAMP, + staging_cleaned = FALSE + FROM upgraded_jobs j + WHERE a.id = j.attachment_id + RETURNING a.id +) +INSERT INTO attachment_outbox (id, job_id, attempt) +SELECT gen_random_uuid(), j.id, j.attempts + 1 +FROM upgraded_jobs j +JOIN queued_attachments a ON a.id = j.attachment_id; + +DELETE FROM attachment_variants v +USING attachments a +WHERE v.attachment_id = a.id AND a.status = 'queued'; diff --git a/src/main/resources/migrations/V9__attachment_outbox.sql b/src/main/resources/migrations/V9__attachment_outbox.sql new file mode 100644 index 00000000..81b1be8d --- /dev/null +++ b/src/main/resources/migrations/V9__attachment_outbox.sql @@ -0,0 +1,23 @@ +CREATE TABLE attachment_outbox ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + attempt INTEGER NOT NULL CHECK (attempt > 0), + available_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + lease_token UUID, + lease_until TIMESTAMPTZ, + CHECK ((lease_token IS NULL) = (lease_until IS NULL)) +); + +CREATE INDEX attachment_outbox_due_idx ON attachment_outbox (available_at, id); + +-- Recover both pending lab4 imports and unfinished work already accepted by lab5. +INSERT INTO attachment_outbox (id, job_id, attempt, available_at) +SELECT gen_random_uuid(), + j.id, + CASE WHEN j.lease_until IS NULL THEN j.attempts + 1 ELSE j.attempts END, + GREATEST(CURRENT_TIMESTAMP, j.next_attempt_at, j.lease_until) +FROM attachment_jobs j +JOIN attachments a ON a.id = j.attachment_id +WHERE a.status IN ('queued', 'processing'); + +ALTER TABLE attachment_jobs DROP COLUMN legacy_dispatch_pending; diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt index 400a8e2e..5a1f20f6 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt @@ -66,7 +66,7 @@ class AttachmentControllerTest { val ready = client.awaitReady(attachment.id, bob) - assertEquals(listOf("thumbnail"), ready.variants.map { it.name }) + assertEquals(setOf("thumbnail", "crop", "optimized"), ready.variants.map { it.name }.toSet()) val path = "/api/v1/attachments/${attachment.id}" val download = client.get("$path/content") { auth(bob) } @@ -88,10 +88,16 @@ class AttachmentControllerTest { ImageIO.read(input) } } - val missingVariant = client.get("$path/content?variant=crop") { auth(bob) } + val crop = client.get("$path/content?variant=crop") { auth(bob) } + val optimized = client.get("$path/content?variant=optimized") { auth(bob) } + val missingVariant = client.get("$path/content?variant=unknown") { auth(bob) } assertEquals(32, decoded.width) assertEquals(20, decoded.height) + assertEquals(HttpStatusCode.OK, crop.status) + assertEquals(ContentType.Image.JPEG, crop.contentType()?.withoutParameters()) + assertEquals(HttpStatusCode.OK, optimized.status) + assertEquals(ContentType.parse("image/webp"), optimized.contentType()?.withoutParameters()) assertEquals(HttpStatusCode.NotFound, missingVariant.status) val outsiderChat = client.createChat(outsider, "outsider", listOf(outsider.id)) diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt index 32d6ca01..21a2213c 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt @@ -7,9 +7,9 @@ import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel import com.mtusi.messenger.main.infrastructure.models.attachment.UploadJournalModel import com.mtusi.messenger.main.infrastructure.repositories.attachment.ExposedAttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.ExposedUploadJournalRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentOutboxRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.ExposedAttachmentJobRepository import com.mtusi.messenger.main.infrastructure.repositories.chat.ExposedChatRepository -import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.infrastructure.services.attachment.storage.S3ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ApiException @@ -81,12 +81,12 @@ class AttachmentCleanupTest { bytes = bytes, contentType = attachment.contentType ) - storage.putThumbnail(id = attachment.id, bytes = bytes) + storage.putVariant(id = attachment.id, name = "thumbnail", bytes = bytes) cleanup.cleanup() assertContentEquals(bytes, storage.get(attachment.id)) - assertContentEquals(bytes, storage.getThumbnail(attachment.id)) + assertContentEquals(bytes, storage.getVariant(attachment.id, "thumbnail")) suspendedTransaction { AttachmentModel.update(where = { AttachmentModel.id eq attachment.id }) { @@ -98,7 +98,7 @@ class AttachmentCleanupTest { assertTrue(storage.ready()) assertFailsWith { storage.get(attachment.id) } - assertFailsWith { storage.getThumbnail(attachment.id) } + assertFailsWith { storage.getVariant(attachment.id, "thumbnail") } } @Test @@ -126,8 +126,11 @@ class AttachmentCleanupTest { config = workerConfig ) - val jobs = ExposedAttachmentJobRepository(attachments) - val broker = application.get() + val outbox = application.get() + val jobs = ExposedAttachmentJobRepository( + attachmentRepository = attachments, + outboxRepository = outbox + ) val file = withContext(Dispatchers.IO) { Files.createTempFile("cleanup-test-", ".png") @@ -162,7 +165,7 @@ class AttachmentCleanupTest { chatRepository = ExposedChatRepository(), objectStorageService = failingStorage, jobRepository = jobs, - jobBrokerService = broker + outboxRepository = outbox ) assertFailsWith { @@ -194,7 +197,7 @@ class AttachmentCleanupTest { chatRepository = ExposedChatRepository(), objectStorageService = storage, jobRepository = jobs, - jobBrokerService = broker + outboxRepository = outbox ) val published = service.upload( chatId = chat.id, diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt index 241d2653..be297788 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt @@ -7,9 +7,14 @@ import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob import com.mtusi.messenger.main.infrastructure.dto.Message import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobReplayModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentOutboxModel import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.JobClaim +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.JobReplayResult +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.AttachmentOutboxDispatchService import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ApiException @@ -51,8 +56,10 @@ import kotlin.test.assertTrue import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.longLiteral import org.jetbrains.exposed.v1.datetime.CurrentTimestamp +import org.jetbrains.exposed.v1.jdbc.select import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update import org.koin.ktor.ext.get @@ -119,7 +126,9 @@ class AttachmentPipelineTest { assertEquals("ready", ready.status) assertEquals(null, ready.errorCode) - val variant = ready.variants.single() + assertEquals(setOf("thumbnail", "crop", "optimized"), ready.variants.map { it.name }.toSet()) + + val variant = ready.variants.single { it.name == "thumbnail" } assertEquals("thumbnail", variant.name) assertEquals("image/jpeg", variant.contentType) @@ -229,7 +238,8 @@ class AttachmentPipelineTest { token = firstToken, errorCode = "storage_unavailable", permanent = false, - maxAttempts = 2 + maxAttempts = 2, + retryDelay = 5.seconds ) } val pendingRetry = client.get(path) { auth(user) }.body() @@ -258,7 +268,23 @@ class AttachmentPipelineTest { leaseSeconds = 60 ) } + val earlyClaim = suspendedTransaction { + jobs.claim( + job = retry, + token = UUID.randomUUID(), + maxAttempts = 2, + leaseSeconds = 60 + ) + } + + assertEquals(JobClaim.Complete, staleClaim) + assertEquals(JobClaim.Complete, earlyClaim) + val retryClaim = suspendedTransaction { + AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { + it[nextAttemptAt] = CurrentTimestamp + longLiteral(-1) + } + jobs.claim( job = retry, token = secondToken, @@ -267,7 +293,6 @@ class AttachmentPipelineTest { ) } - assertEquals(JobClaim.Retry(2), staleClaim) assertEquals(JobClaim.Acquired(2), retryClaim) val claimedJob = suspendedTransaction { @@ -308,8 +333,53 @@ class AttachmentPipelineTest { assertEquals(HttpStatusCode.Conflict, unavailableContent.status) assertEquals(HttpStatusCode.Conflict, rejectedMessage.status) - assertTrue(application.get().process(job)) + val worker = application.get() + + assertTrue(worker.process(job)) assertEquals(failed, client.get(path) { auth(user) }.body()) + + val failedJob = suspendedTransaction { + jobs.findFailed(limit = 100).single { it.jobId == job.jobId } + } + + assertTrue(failedJob.replayable) + assertEquals("retry_exhausted", failedJob.errorCode) + assertEquals(2, failedJob.attempts) + + val replayResult = suspendedTransaction { jobs.replay(job.jobId, "integration-test") } + + assertEquals(JobReplayResult.QUEUED, replayResult) + assertTrue(worker.process(retry)) + assertEquals("queued", client.get(path) { auth(user) }.body().status) + + // The new run gets a fresh budget, while the absolute envelope number keeps increasing. + val replayedJob = job.copy(attempt = 3) + val replayToken = UUID.randomUUID() + val replayClaim = suspendedTransaction { + jobs.claim( + job = replayedJob, + token = replayToken, + maxAttempts = 1, + leaseSeconds = 60 + ) + } + + assertEquals(JobClaim.Acquired(3, attemptInRun = 1), replayClaim) + assertEquals( + JobReplayResult.NOT_FAILED, + suspendedTransaction { jobs.replay(job.jobId, "integration-test") } + ) + + val audit = suspendedTransaction { + AttachmentJobReplayModel + .selectAll() + .where(AttachmentJobReplayModel.jobId eq job.jobId) + .single() + } + + assertEquals("integration-test", audit[AttachmentJobReplayModel.operator]) + assertEquals(3, audit[AttachmentJobReplayModel.attempt]) + assertEquals("retry_exhausted", audit[AttachmentJobReplayModel.previousErrorCode]) } @Test @@ -347,6 +417,7 @@ class AttachmentPipelineTest { val next = upload("next") val broker = application.get() val worker = application.get() + val dispatcher = application.get() suspend fun metadata(attachment: Attachment): Attachment { return client.get("/api/v1/attachments/${attachment.id}") { auth(user) } @@ -362,6 +433,12 @@ class AttachmentPipelineTest { } coroutineScope { + val publisher = launch { + while (true) { + dispatcher.publishPending() + delay(50.milliseconds) + } + } val consumer = launch { broker.consume(worker::process) } @@ -374,11 +451,20 @@ class AttachmentPipelineTest { AttachmentJobModel.update(where = { AttachmentJobModel.id eq busyJob.jobId }) { it[leaseUntil] = CurrentTimestamp + longLiteral(-1) } + AttachmentOutboxModel.update( + where = { + (AttachmentOutboxModel.jobId eq busyJob.jobId) and + (AttachmentOutboxModel.attempt eq 1) + } + ) { + it[availableAt] = CurrentTimestamp + longLiteral(-1) + } } awaitReady(busy) } finally { consumer.cancelAndJoin() + publisher.cancelAndJoin() } } } @@ -405,10 +491,10 @@ class AttachmentPipelineTest { val calls = AtomicInteger() val processor = object : ImageProcessingService { - override suspend fun thumbnail( + override suspend fun variants( bytes: ByteArray, declaredType: String - ): ProcessedImage { + ): Map { calls.incrementAndGet() if (permanent) { @@ -424,16 +510,37 @@ class AttachmentPipelineTest { attachmentRepository = application.get(), objectStorageService = application.get(), imageProcessingService = processor, - jobBrokerService = application.get(), config = SerialWorkerConfig(maxAttempts = 2) ) + val startedAt = suspendedTransaction { + AttachmentJobModel + .select(CurrentTimestamp) + .where(AttachmentJobModel.id eq job.jobId) + .single()[CurrentTimestamp] + } + assertTrue(service.process(job)) if (!permanent) { val pendingRetry = client.get(path) { auth(user) }.body() assertEquals("queued", pendingRetry.status) + + val availableAt = suspendedTransaction { + AttachmentJobModel + .select(AttachmentJobModel.nextAttemptAt) + .where(AttachmentJobModel.id eq job.jobId) + .single()[AttachmentJobModel.nextAttemptAt] + } + assertTrue(assertNotNull(availableAt) >= startedAt + 500.milliseconds) + + // Advance the stored deadline instead of waiting on wall clock in the test. + suspendedTransaction { + AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { + it[nextAttemptAt] = CurrentTimestamp + longLiteral(-1) + } + } assertTrue(service.process(job.copy(attempt = 2))) } @@ -447,6 +554,24 @@ class AttachmentPipelineTest { assertTrue(service.process(job)) assertEquals(expectedCalls, calls.get(), "terminal redelivery must not run the processor again") + + val jobs = application.get() + + if (permanent) { + assertEquals( + JobReplayResult.PERMANENT_FAILURE, + suspendedTransaction { jobs.replay(job.jobId, "integration-test") } + ) + } else { + val replay = suspendedTransaction { + AttachmentModel.update(where = { AttachmentModel.id eq queued.id }) { + it[storageCleanupStarted] = true + } + jobs.replay(job.jobId, "integration-test") + } + + assertEquals(JobReplayResult.SOURCE_EXPIRED, replay) + } } } diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt index 65783a11..08c8d2d2 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt @@ -25,12 +25,102 @@ import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertNull import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds class ImageProcessingServiceTest { private val service = DefaultImageProcessingService() private val colors = listOf(Color.RED, Color.GREEN, Color.BLUE, Color.YELLOW) + @Test + fun `published variants apply orientation and preserve the centered crop`() = runBlocking { + + val source = BufferedImage(1600, 800, BufferedImage.TYPE_INT_RGB) + val graphics = source.createGraphics() + + try { + graphics.color = Color.RED + graphics.fillRect(0, 0, 400, 800) + graphics.color = Color.GREEN + graphics.fillRect(400, 0, 800, 800) + graphics.color = Color.BLUE + graphics.fillRect(1200, 0, 400, 800) + } finally { + graphics.dispose() + } + + val bytes = withPngExif(encode(source, "png"), 6) + val variants = service.variants(bytes, "image/png") + + assertEquals(setOf("thumbnail", "crop", "optimized"), variants.keys) + assertEquals(128 to 256, variants.getValue("thumbnail").let { it.width to it.height }) + + val crop = variants.getValue("crop") + val decodedCrop = decode(crop.bytes) + + assertEquals("image/jpeg", crop.contentType) + assertEquals(128 to 128, decodedCrop.width to decodedCrop.height) + assertColor(Color.GREEN, Color(decodedCrop.getRGB(8, 8))) + assertColor(Color.GREEN, Color(decodedCrop.getRGB(120, 120))) + + val optimized = variants.getValue("optimized") + val decodedOptimized = decodeWebp(optimized.bytes) + + assertEquals("image/webp", optimized.contentType) + assertEquals(640 to 1280, optimized.width to optimized.height) + assertEquals(640 to 1280, decodedOptimized.width to decodedOptimized.height) + assertColor(Color.RED, Color(decodedOptimized.getRGB(250, 100))) + assertColor(Color.GREEN, Color(decodedOptimized.getRGB(250, 640))) + assertColor(Color.BLUE, Color(decodedOptimized.getRGB(250, 1180))) + assertFalse(webpChunks(optimized.bytes).any { it in setOf("EXIF", "XMP ", "ICCP") }) + } + + @Test + fun `optimized WebP preserves alpha and never enlarges a small source`() = runBlocking { + + val source = BufferedImage(80, 40, BufferedImage.TYPE_INT_ARGB) + val graphics = source.createGraphics() + + try { + graphics.color = Color(255, 0, 0, 128) + graphics.fillRect(0, 0, 40, 40) + } finally { + graphics.dispose() + } + + val optimized = service.variants(encode(source, "png"), "image/png").getValue("optimized") + val decoded = decodeWebp(optimized.bytes) + + assertEquals(80 to 40, optimized.width to optimized.height) + assertEquals(80 to 40, decoded.width to decoded.height) + assertEquals(128, Color(decoded.getRGB(20, 20), true).alpha) + assertEquals(0, Color(decoded.getRGB(60, 20), true).alpha) + } + + @Test + fun `worker subprocess preserves permanent image failures`() = runBlocking { + + val invalidImage = assertFailsWith { + service.variants("not an image".toByteArray(), "image/png") + } + + assertEquals(ErrorCode.UNSUPPORTED_MEDIA_TYPE, invalidImage.code) + } + + @Test + fun `image subprocess stops at its deadline`() = runBlocking { + + assertFailsWith { + runImageProcess( + command = listOf("sleep", "10"), + input = ByteArray(0), + outputLimit = 16, + timeout = 100.milliseconds + ) + } + } + @Test fun `thumbnail fits the box floors dimensions and never enlarges the source`() = runBlocking { @@ -222,6 +312,46 @@ class ImageProcessingServiceTest { } } + private suspend fun decodeWebp( + bytes: ByteArray + ): BufferedImage { + + val decoded = runImageProcess( + command = listOf("dwebp", "-quiet", "-o", "-", "--", "-"), + input = bytes, + outputLimit = MAX_ATTACHMENT_BYTES, + timeout = 10.seconds + ) + + assertEquals(0, decoded.exitCode) + + return decode(decoded.output) + } + + private fun webpChunks( + bytes: ByteArray + ): List { + + assertEquals("RIFF", String(bytes, 0, 4, Charsets.US_ASCII)) + assertEquals("WEBP", String(bytes, 8, 4, Charsets.US_ASCII)) + + return buildList { + var offset = 12 + + while (offset < bytes.size) { + add(String(bytes, offset, 4, Charsets.US_ASCII)) + + val size = ByteBuffer.wrap(bytes, offset + 4, 4) + .order(ByteOrder.LITTLE_ENDIAN) + .int + + offset += 8 + size + (size and 1) + } + + assertEquals(bytes.size, offset) + } + } + private fun tiff(orientation: Int): ByteArray { return ByteBuffer.allocate(26) .order(ByteOrder.LITTLE_ENDIAN) diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt index ca127059..2c9f84ec 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt @@ -75,8 +75,8 @@ class LegacyAttachmentMigrationTest { attachment } - val variant = ready.variants.single() - val thumbnail = storage.getThumbnail(id) + val variant = ready.variants.single { it.name == "thumbnail" } + val thumbnail = storage.getVariant(id = id, name = "thumbnail") assertContentEquals(original, storage.get(id)) assertEquals("thumbnail", variant.name)