diff --git a/.env.example b/.env.example index 3d65fa79..a0b3ae35 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,13 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_api_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +# The implementation uses structured fields instead of a credential-bearing URL. +BROKER_HOST=broker +BROKER_PORT=5672 +BROKER_USER=messenger +BROKER_PASSWORD_FILE=/run/secrets/broker_password +BROKER_QUEUE=attachment.process.v1 +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 diff --git a/REPORT.md b/REPORT.md index 33ade9d4..f0cbd128 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,36 +1,18 @@ # Отчёт — лабораторная 5 - Автор, группа: Кучерявенко Максим, БВТ2303. -- Текущий этап: перенос работающей основы lab4 для просмотра перед выполнением lab5. -- Целевая оценка: будет выбрана при переходе к реализации; работа ведётся поэтапно, начиная с уровня 3. +- Целевая оценка: **3**. Уровни 4 и 5 не заявляются. +- Основа: собственный проект lab4 на Kotlin 2.3.20, Ktor 3.3.0 и JDK 21. Архитектура controller → service → repository, Koin/KSP и Exposed 1.0.0 сохранены; стиль сверяется с существующими репозиториями проекта и Pixefy. +- Новые зависимости: RabbitMQ Java Client 5.26.0 и metadata-extractor 2.19.0. Библиотека Java Client предоставляет AMQP API, поэтому здесь используются её Java configuration objects; операции PostgreSQL написаны на Exposed DSL, S3 — на AWS SDK for Kotlin DSL. - Commit/PR: после просмотра и публикации. -- Основа: Kotlin 2.3.20, Ktor 3.3.0, JDK 21, Koin/KSP, Exposed 1.0.0, PostgreSQL, Redis и MinIO. Существующие архитектура controller → service → repository, DSL и стиль кода сохранены; ориентир — проекты Pixefy. -- Источники: исходники собственного lab4, задание и контракты lab5, проекты Pixefy. ИИ использован для переноса и проверки структуры проекта. +- ИИ использован как парный разработчик при реализации, ревью, подготовке тестов и отчёта. -## Что перенесено +## Запуск из чистого clone -- Исходники приложения и 80 JVM-тестов, конфигурации, Gradle wrapper и зависимости. -- Миграции Flyway V1–V5: пользователи, чаты, сообщения, сессии, идемпотентность, вложения и журнал загрузок. -- Bearer-аутентификация, Redis-кэш с fallback в PostgreSQL, keyset-пагинация и идемпотентная отправка сообщений. -- Синхронные upload/download PNG/JPEG, приватный bucket, проверки доступа и изображений, временные файлы и очистка по журналу. -- Dockerfile, Compose с двумя API-репликами, HAProxy, PostgreSQL, Redis, MinIO и повторяемым bootstrap. -- Скрипты запуска, проверки S3-прав, прерывания загрузки, нагрузки, переключения реплик и перезапуска стенда. -- Проектные инструкции AGENTS.md и локальный skill Kotlin/Exposed для дальнейшего соблюдения архитектуры и стиля. - -Имя Compose/Gradle-проекта изменено на `mtusi-messenger-lab5`, образ приложения — на `mtusi-messenger:lab5`. Это отделяет контейнеры, сети, named volumes и образ от lab4. Оригиналы и записи БД из работающего стенда lab4 не переносились: lab5 получает собственные volumes при запуске. - -Исходные README задания, lab.json, contracts/, публичные tests/, examples/, CI и шаблон PR лабораторной 5 сохранены. В `.env.example` сохранены новые настройки broker/worker/jobs/staging; пути S3 credentials согласованы с отдельным пользователем API. - -Исторические материалы lab4 находятся в [archive/lab4](archive/lab4/README.md): исходный отчёт, evidence и документы к защите. Их результаты не являются доказательствами выполнения lab5. - -## Запуск перенесённой основы из чистого clone - -Нужны Docker с Compose v2, Python 3.9+ и openssl. Для Gradle-проверок — JDK 21 и Docker для Testcontainers. +Нужны Docker с Compose v2, Python 3.9+, openssl. JVM-тесты требуют JDK 21 и работающий Docker: Testcontainers поднимает отдельные PostgreSQL, Redis, MinIO и RabbitMQ. ```sh ./scripts/init-secrets.sh - -# Отдельный порт, чтобы не мешать lab4 на 8084 и другим лабораторным. export LAB_HTTP_PORT=8085 export BASE_URL=http://localhost:8085 @@ -40,57 +22,145 @@ docker compose ps make check ./gradlew test +make test -# Безопасная остановка с сохранением данных: +# Основной сценарий lab5: worker остановлен, 202 принят, +# обе API и broker перезапущены, тот же файл доходит до ready. +python3 scripts/attachment-worker-recovery.py + +# Доступ через другую API-реплику и приватность объекта. +python3 scripts/attachment-failover.py + +# Сохранность после перезапуска всех компонентов, включая worker/broker. +make test-persistence ACTION_SCRIPT=scripts/restart.sh + +# Безопасная остановка с сохранением volumes: docker compose down # Только для намеренного удаления всех учебных данных lab5: # docker compose down -v ``` -Секреты создаются заново локальным скриптом; секреты lab4, локальная `.env`, runtime-конфигурации, build/cache и `.git` не копировались. `./gradlew run` создаёт недостающие конфигурации из `run/_template`, сохраняя существующие локальные настройки. Для запуска вне Compose необходимо отдельно настроить доступные с host PostgreSQL, Redis и S3. +Секреты генерируются локально, существующие файлы сохраняются. В Git не включаются `.env`, `secrets/`, `run/development/`, данные volumes и build/cache. Проект Compose называется `mtusi-messenger-lab5`: его контейнеры, сеть и volumes отделены от lab4. Наружу опубликован только HAProxy на loopback; PostgreSQL, Redis, S3 и RabbitMQ доступны внутри сети стенда. -### Настройки инфраструктуры +Для запуска через `./gradlew run` нужны отдельные локальные настройки подключений в `run/development/config`. Шаблон не является готовой конфигурацией существующего PostgreSQL на компьютере; роль `username` необходимо заменить реальной. Автоматически менять локальную БД или её пользователей задача `run` не будет. Воспроизводимый путь проверки этой работы — Compose. -- `LAB_HTTP_PORT` меняет только опубликованный порт HAProxy; внутри API остаётся 8080. В Compose по умолчанию две API-реплики; их число задаётся `API_REPLICAS`. -- `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` и `S3_BUCKET` передаются через Compose. Доступ к PostgreSQL и Redis использует существующие `DB_*` и `REDIS_*`, а не предложенные skeleton `DATABASE_URL_FILE` и `CACHE_URL_FILE`. -- API обращается к `http://s3:9000`; `S3_ACCESS_KEY_ID_FILE` и `S3_SECRET_ACCESS_KEY_FILE` указывают на secret-файлы прикладного S3-пользователя. Административные credentials доступны только MinIO и bootstrap. -- Образы и зависимости сохранены из lab4: PostgreSQL 17.6, Redis 8.4.0, HAProxy 3.2.4; MinIO RELEASE.2025-09-07T16-13-09Z и mc RELEASE.2025-08-13T08-35-41Z закреплены digest. API имеет лимиты 1 CPU / 1 GiB, временный `/tmp` — tmpfs 32 MiB. -- `S3_REGION=us-east-1`, пределы 10 MiB и 40 млн пикселей закреплены в реализации. `APP_ENV` и переключатель `SESSION_STORE` не меняют реализацию. -- Переменные `BROKER_URL_FILE`, `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS` пока являются частью skeleton и не используются перенесённым приложением. +## Сервисы и настройки -## Архитектура текущего этапа +| Роль | Compose service | Версия / хранение | +| --- | --- | --- | +| Балансировщик | lb | HAProxy 3.2.4; внешний порт LAB_HTTP_PORT, по умолчанию 8080 | +| HTTP API | api × 2 | Образ mtusi-messenger:lab5, собранный из исходников | +| Обработчик | worker | Тот же образ, отдельный процесс с командой `worker` | +| Метаданные и задания | postgres | PostgreSQL 17.6; postgres-data | +| Кэш сессий | redis | Redis 8.4.0; без persistence, источник истины — PostgreSQL | +| Объектное хранилище | s3 | MinIO RELEASE.2025-09-07T16-13-09Z; s3-data | +| Durable очередь | broker | RabbitMQ 4.1.8-alpine; broker-data, стабильный hostname broker | -Клиент → HAProxy → API × 2 → PostgreSQL + Redis + MinIO. +MinIO и mc закреплены digest из lab4. RabbitMQ также закреплён multi-platform digest `sha256:cd4009416f83d7e5895020cd71b0963f3b8a4dfc3de74bbb6fb3bfe5f81f01e0`. Одноразовый `migrate` применяет миграции Flyway V1–V8 до старта API и worker. `s3-init` создаёт приватный bucket и policy и допускает повторный запуск. -Это основа lab4: upload возвращает **201 ready** после S3 PUT и COMMIT метаданных. Фоновой обработки изображений нет. Из фоновых задач перенесён только существующий уборщик незавершённых загрузок, который не является worker lab5. +Переменные `.env.example` сопоставлены реализации так: -## Проверки переноса +- `API_REPLICAS`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS`, `S3_BUCKET` и `LAB_HTTP_PORT` управляют существующим стендом. +- PostgreSQL и Redis используют `DB_*` и `REDIS_*` с password-файлами. `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `APP_ENV` и переключатель `SESSION_STORE` из skeleton не управляют выбранной реализацией. +- `BROKER_HOST`, `BROKER_PORT`, `BROKER_USER`, `BROKER_PASSWORD_FILE`, `BROKER_QUEUE` задают RabbitMQ. Вместо `BROKER_URL_FILE` используются отдельные поля, чтобы URL с credentials не попадал в логи. Имя очереди по умолчанию — `attachment.process.v1`. +- `WORKER_CONCURRENCY=1` задаёт число последовательных consumer-циклов worker; допустимы 1–4. Каждый цикл держит не более одной unacked delivery. Проверенный профиль использует 1; увеличение требует отдельной оценки памяти. +- `JOB_MAX_ATTEMPTS=5` ограничивает количество захватов работы, `JOB_TIMEOUT_SECONDS=60` задаёт coroutine timeout, lease устанавливается на timeout + 15 секунд. +- `STAGING_RETENTION_SECONDS=86400` задаёт задержку удаления staging для terminal-состояний; минимум 3600. Для queued/processing исходник не удаляется по возрасту. +- Регион S3 фиксирован `us-east-1`; пределы 10 MiB и 40 млн пикселей закреплены в коде и не увеличиваются переменными окружения. -Проверки переноса выполнены 16.11.2026: +## Архитектура и граница 202 -| Проверка | Результат | -| --- | --- | -| `rtk gradlew compileKotlin compileTestKotlin --console=plain` | BUILD SUCCESSFUL; исходники приложения и всех JVM-тестов компилируются. | -| `rtk gradlew test --tests '*ImageValidatorTest' --tests '*TemporaryUploadTest' --tests '*DefaultHashingServiceTest' --console=plain` | 9 тестов, 0 failures/errors/skipped. | -| `make check` | PASS: структура skeleton, локальные ссылки и Python-синтаксис. | -| `docker compose config --quiet` | Конфигурация корректна; это не запуск контейнеров. | -| Сравнение с исходным lab4 | 176 файлов src/, gradle/, run/_template/, infra/haproxy/ и infra/s3/ совпадают побайтно; executable bit скриптов сохранён. | -| Сохранность skeleton lab5 | 32 исходных файла задания, контрактов, публичных проверок и связанных материалов не изменены. | +Клиент → HAProxy → API → PostgreSQL + приватный staging S3 → RabbitMQ → worker → final S3 + PostgreSQL. -Краткий протокол: [evidence/transfer-checks.txt](evidence/transfer-checks.txt). +API принимает весь файл, проверяет доступ к чату, filename, MIME, размер и корректность изображения. HTTP-тело читается порциями по 8192 bytes во временный файл с лимитом 10 MiB и таймаутом 15 секунд; семафор допускает одну активную загрузку на API-реплику. Для сохранения обязательных проверок lab4 перед 202 выполняется полная валидация ImageIO, включая ограничения пикселей, CRC/IEND PNG и EOI JPEG. Resize и публикация вариантов выполняются только worker. -Docker daemon сейчас недоступен. Полный набор Testcontainers-тестов и сам Compose-стенд в этой проверке не запускались. Исторические 80 успешных тестов в archive/lab4/evidence относятся к lab4, а не к новому прогону lab5. Эти проверки переноса не подтверждают асинхронный pipeline lab5. +Далее API: -`make test` по неизменённому lab.json запускает публичный профиль **lab5**. Он требует 202 queued и thumbnail, поэтому пока не используется как успешная проверка перенесённой основы. После начала выполнения lab5 проверяться будет именно этот контракт; менять его на профиль lab4 не требуется. +1. Регистрирует намерение загрузки в унаследованном журнале `attachment_uploads`. +2. Сохраняет неизменённые bytes в `staging/` в MinIO. +3. В одной DB-транзакции создаёт Attachment со статусом queued и строку `attachment_jobs`, затем удаляет завершённое намерение из журнала. +4. Публикует persistent JSON в durable очередь и ожидает publisher confirm. +5. Возвращает **202** со снимком queued, пустыми variants и error_code=null. -## Следующий этап после просмотра +Worker может успеть завершить работу до первого GET, но ответ POST остаётся снимком queued. Подтверждённая 202 работа уже имеет staging, метаданные, запись задания и подтверждённое брокером сообщение. Локальный временный файл после этого не нужен. -Реализация ещё не начата: предстоит выбрать durable broker, добавить отдельный worker, устойчивый staging и задания, состояния queued/processing/ready/failed, thumbnail, ограниченные повторы и идемпотентность обработки. Затем будут выполнены проверки уровня 3. Пункты уровней 4 и 5 рассматриваются отдельно после предыдущего этапа. +При ошибке публикации 202 не возвращается. Если строка ещё queued, она переводится в failed с `enqueue_failed`; уже захваченная или завершённая работа не откатывается. После жёсткого crash между COMMIT и publish может остаться неподтверждённая queued-запись без сообщения: автоматическое восстановление этого окна относится к outbox уровня 4 и здесь не заявляется. -## Ограничения +## Договор producer/worker и состояния -- Контракт lab5 ещё не реализован; готовность на оценку 3/4/5 не заявляется. -- Унаследованные attachment load/failover-скрипты ожидают синхронный 201. При переходе на 202 их потребуется адаптировать вместе с реализацией. -- Валидация использует ограниченный массив bytes и полный raster ImageIO; крайний случай памяти очень широких PNG остаётся ограничением исходной основы. -- PostgreSQL, MinIO и HAProxy одноузловые. Volumes обеспечивают сохранность при restart, но не заменяют backup и репликацию. -- Журнал lab4 очищает учтённые незавершённые загрузки, а не выполняет глобальную сверку bucket. Записи abandoned сохраняются для удаления поздних PUT и могут накапливаться. +`AttachmentJob` соответствует обязательным полям `contracts/jobs.schema.json`: schema_version=1, job_id, type=attachment.process.v1, attachment_id, pipeline_version=1, attempt и created_at. В сообщении нет bytes, credentials, пользовательского пути или URL. Worker получает доверенные метаданные по ID из PostgreSQL. + +Логическая job_id стабильна. Broker redelivery сохраняет envelope. Управляемый retry публикует сообщение той же job с увеличенным attempt. Worker подтверждает старую доставку после подтверждения новой публикации. Если lease ещё действует, worker отправляет тот же envelope в конец очереди, сохраняя номер попытки. Если старый envelope приходит после изменения DB-счётчика, worker публикует актуальную попытку без повторной обработки. + +- queued → processing: worker блокирует запись задания, проверяет статус и lease, увеличивает счётчик попыток и записывает случайный token. +- processing → ready: после записи original и thumbnail одна DB-транзакция публикует вариант и состояние ready, затем снимает lease. +- processing → queued: при временной ошибке с оставшимися попытками PostgreSQL сохраняет внутренний код причины и время следующего запуска. Повтор можно запустить сразу, поэтому используется время БД в момент перехода. Затем worker публикует retry и ждёт подтверждения. +- processing → failed: постоянная ошибка изображения или исчерпание попыток. В API остаётся безопасный error_code, без stack trace и секретов. +- ready/failed + повторная delivery: no-op с ack. Старый token не может опубликовать результат после нового захвата. + +Default consumer использует manual `basicGet`: следующий запрос выполняется после завершения предыдущего handler. Это явно ограничивает число delivery в работе; значение basicQos(1) не используется как единственная гарантия для polling API. Ack отправляется после устойчивого результата или подтверждённой передачи следующей попытки. При остановке/обрыве unacked delivery возвращается в очередь. Истёкший lease допускает новый захват; попытки по-прежнему ограничены DB-счётчиком. + +Это доставка **at least once** с идемпотентным эффектом, а не exactly-once. Объекты имеют детерминированные ключи, а вариант в БД — первичный ключ `(attachment_id, name)`. В этом этапе используется только pipeline_version=1. + +## Worker и изображения + +Worker запускается отдельным процессом и не принимает бизнес-HTTP-запросы. Служебные health endpoints нужны Compose. Остановка worker не переводит resize в API: новые вложения остаются queued, текстовые сообщения продолжают работать, queued-вложение разрешено прикреплять к сообщению. + +Worker проверяет размер и SHA-256 staging, один раз декодирует изображение и строит thumbnail. Для записи оригинала в S3 worker использует уже проверенные байты: + +- `originals/` — исходные bytes без перекодирования; +- `variants//v1/thumbnail.jpg` — JPEG, вписанный в 256×256 без увеличения; размеры округляются вниз, минимум 1 px. + +EXIF orientation 1–8 применяется при отрисовке. Alpha компонуется на белый. JPEG кодируется стандартным ImageIO writer; чужие metadata в новый JPEG не переносятся. Fixture 320×200 даёт thumbnail 256×160. Метаданные варианта содержат собственные MIME, размеры, размер bytes и SHA-256. + +Для queued/processing/failed скачивание content возвращает 409. Для ready доступен original и thumbnail, отсутствующий вариант получает 404. Чужой пользователь получает 404 до выдачи метаданных или содержимого. Failed-вложение нельзя добавить в новое сообщение, но уже созданное сообщение корректно воспроизводится по прежнему Idempotency-Key, даже если вложение позднее стало failed. + +При обновлении с lab4 миграция V8 переводит вложения без thumbnail в queued и создаёт задания для уже сохранённых оригиналов. Worker снимает отметку об ожидающей публикации после подтверждения RabbitMQ. Если процесс остановится раньше, он повторит публикацию того же задания. Существующий объект читается по прежнему ключу и сохраняет исходные байты. Скачивание снова доступно после готовности thumbnail. + +## Сохранность, cleanup и безопасность + +Очередь durable classic, сообщения persistent, публикации используют mandatory routing и publisher confirms. RabbitMQ сохраняет данные в отдельном volume. Producer confirm означает приём брокером, consumer ack — устойчивый эффект worker; эти подтверждения не взаимозаменяемы. + +Унаследованный журнал закрывает orphan-окно между staging PUT и записью метаданных. После часового grace period уборщик помечает просроченные намерения abandoned и удаляет связанные ключи. Записи сохраняются для повторного удаления позднего PUT. По истечении retention уборщик удаляет staging для ready и все записанные файлы для failed. У вложений из lab4 сохраняется оригинал. Отметка staging_cleaned фиксирует завершение уборки. Очистка обходит только terminal-состояния. + +Root S3 credentials доступны только MinIO/bootstrap. Прикладная policy дополнена разрешениями для staging/* и variants/*, сохраняя запрет административных операций и записи вне рабочих prefixes. S3 и console не опубликованы на host, console привязана к loopback внутри контейнера. Пароль RabbitMQ читается из secret-файла; брокер также не опубликован наружу. + +При завершении приложения явно отменяются consumer, cleanup и синхронизация revocation, затем закрываются подключения. Это предотвращает работу старых фоновых циклов после закрытия DI/БД и их вмешательство в другие экземпляры тестового приложения. + +## Проверки уровня 3 + +Стендовые проверки из таблицы выполнены **17 ноября 2026 года** на порту 8085. После исправлений **18 ноября** отдельно собран стенд на порту 8086, где повторно прошли публичный smoke и сценарий восстановления worker. Полный JVM-набор также запущен повторно после правок. + +| Проверка | Команда / сценарий | Результат и доказательство | +| --- | --- | --- | +| Структура и контракт | `make check` | PASS; [вывод](evidence/check.txt) | +| Конфигурация и запуск Compose | `docker compose config --quiet`, `docker compose up -d --build --wait` | Все восемь рабочих контейнеров healthy; [конфигурация и состояние стенда](evidence/compose.txt) | +| JVM-регрессии и новый pipeline | `rtk gradlew compileKotlin compileTestKotlin test --console=plain` | **94 теста, 0 ошибок и пропусков**, включая обновление с lab4, очистку частичного результата и обработку следующей задачи при действующем lease; [сводка по JUnit XML](evidence/gradle-tests.txt) | +| Публичный сценарий lab5 | `make test BASE_URL=http://localhost:8085` | PASS: upload 202, ready, оригинал и настоящий JPEG thumbnail; [вывод](evidence/smoke.txt) | +| Worker остановлен, API и broker перезапущены | `python3 scripts/attachment-worker-recovery.py` | 202 queued → сообщение создано → content 409 → restart обеих API и broker с volumes → запуск worker → та же запись ready, неизменённый оригинал и JPEG 256×160; [вывод с attachment_id](evidence/worker-recovery.txt) | +| Durable доставка и повтор одной job | `RabbitMqJobBrokerServiceTest`, `AttachmentPipelineTest` в JVM-наборе | Реальный restart контейнера RabbitMQ сохраняет подтверждённое сообщение; nack возвращает тот же envelope; повтор завершённой job сохраняет прежний результат | +| Ограниченные попытки и failed | `AttachmentPipelineTest` в JVM-наборе | Busy lease, постоянная ошибка изображения и исчерпание временных попыток; failed запрещает content и новое сообщение, прежний Idempotency-Key воспроизводит созданное сообщение | +| Правила thumbnail | `ImageProcessingServiceTest` в JVM-наборе | Шесть тестов: JPEG/PNG EXIF orientation, alpha на белом, размеры с округлением вниз, отсутствие увеличения и удаление orientation из результата | +| Переключение API-реплики | `python3 scripts/attachment-failover.py` | Загрузившая реплика остановлена, другая отдаёт оригинал с тем же SHA-256; анонимный S3 GET — 403, чужой пользователь — 404; [вывод](evidence/attachment-failover.txt) | +| Привилегии S3 и повторный bootstrap | `scripts/check-s3-permissions.sh` | Запрещены административные операции, изменение policy и запись вне рабочих prefixes; console недоступна из общей сети; [вывод](evidence/s3-permissions.txt) | +| Полный restart с сохранением данных | `make test-persistence BASE_URL=http://localhost:8085 ACTION_SCRIPT=scripts/restart.sh` | PASS: подтверждённые данные и сессия сохранены; StartedAt изменился у обеих API, worker, broker, LB, PostgreSQL, Redis и MinIO; [вывод и времена до/после](evidence/persistence.txt) | + +Для сценариев со стендом установлены `LAB_HTTP_PORT=8085` и `BASE_URL=http://localhost:8085`, как в инструкции запуска выше. Полный JVM-прогон также проверяет сохранённые возможности lab1–lab4: авторизацию, чаты, конкурентную запись, идемпотентность сообщений, межрепличный отзыв сессий, миграции и отказы зависимостей. Исторические материалы lab4 находятся в `archive/lab4/`; они не являются доказательствами lab5. + +## Ограничения и границы следующего этапа + +- Outbox/автоматическое восстановление разрыва COMMIT/publish, backoff с jitter, DLQ/operator replay, crop и optimized WebP относятся к уровню 4 и не реализованы. +- Валидация использует массив ≤10 MiB и полный raster ImageIO; это не полностью потоковый декодер. Очень широкие изображения остаются краевым случаем памяти исходной реализации. Worker ограничен 1 GiB контейнерной памяти; при JVM OOM включён аварийный выход с последующим restart, а число попыток сохраняется в БД. +- Coroutine timeout кооперативный и не гарантирует принудительное прерывание блокирующего декодера в середине вызова. Lease/token предотвращает публикацию устаревшим обработчиком, но не заменяет жёсткое ограничение CPU-времени отдельного декодера. +- PostgreSQL, MinIO, RabbitMQ и HAProxy одноузловые. Volumes сохраняют данные при restart; отказоустойчивость узла и backup не заявляются. +- После crash незавершённая processing-работа может ждать истечения lease. Секундного SLA на восстановление при любом сбое не заявляется. +- Сравнение нескольких pipeline versions и защита от перезаписи результата старой версией относятся к исследовательскому треку; сейчас применяется версия 1. + +## Источники + +- Задание, contracts/README.md, contracts/attachment-states.md и contracts/jobs.schema.json этого репозитория. +- [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability). +- [Publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms). +- [RabbitMQ Java Client API](https://www.rabbitmq.com/client-libraries/java-api-guide). +- [AWS SDK for Kotlin](https://github.com/awslabs/aws-sdk-kotlin). +- [metadata-extractor](https://github.com/drewnoakes/metadata-extractor). diff --git a/build.gradle.kts b/build.gradle.kts index 97d10253..2f3348ba 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,6 +47,8 @@ dependencies { implementation(libs.lettuce) implementation(awssdk.services.s3) + implementation(libs.rabbitmq) + implementation(libs.metadata.extractor) implementation(libs.bouncycastle) diff --git a/compose.yaml b/compose.yaml index c8ce9309..6a3c3a67 100644 --- a/compose.yaml +++ b/compose.yaml @@ -55,6 +55,30 @@ services: memory: 128m restart: unless-stopped + broker: + # RabbitMQ 4.1.8-alpine, pinned multi-platform image. + image: rabbitmq@sha256:cd4009416f83d7e5895020cd71b0963f3b8a4dfc3de74bbb6fb3bfe5f81f01e0 + hostname: broker + entrypoint: ["/bin/sh", "/bootstrap.sh"] + secrets: + - broker_password + volumes: + - broker-data:/var/lib/rabbitmq + - ./infra/rabbitmq/bootstrap.sh:/bootstrap.sh:ro + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512m + restart: unless-stopped + s3: # MinIO RELEASE.2025-09-07T16-13-09Z, pinned multi-platform image. image: quay.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e @@ -141,6 +165,8 @@ services: condition: service_healthy s3-init: condition: service_completed_successfully + broker: + condition: service_healthy environment: APP_HOST: "0.0.0.0" APP_PORT: "8080" @@ -155,17 +181,24 @@ services: REDIS_PASSWORD_FILE: /run/secrets/redis_password SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} SESSION_CACHE_SINGLE_FLIGHT: ${SESSION_CACHE_SINGLE_FLIGHT:-true} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} SHUTDOWN_DRAIN_DELAY_MS: "3000" S3_ENDPOINT: http://s3:9000 S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + BROKER_HOST: broker + BROKER_PORT: "5672" + BROKER_USER: messenger + BROKER_PASSWORD_FILE: /run/secrets/broker_password + BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1} secrets: - db_password - redis_password - s3_api_access_key_id - s3_api_secret_access_key + - broker_password # SIGTERM: readiness 503 and 3 s drain (HAProxy needs <= 2 s to notice), then Ktor closes the socket and # finishes requests in flight within 5 s. 3 + 5 = 8 s < 10 s before Docker sends SIGKILL. tmpfs: @@ -173,6 +206,56 @@ services: stop_grace_period: 10s restart: unless-stopped + worker: + image: mtusi-messenger:lab5 + command: ["worker"] + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + environment: + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + DB_HOST: postgres + DB_PORT: "5432" + DB_NAME: messenger + DB_USER: messenger + DB_PASSWORD_FILE: /run/secrets/db_password + DB_MIGRATE_ON_START: "false" + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + BROKER_HOST: broker + BROKER_PORT: "5672" + BROKER_USER: messenger + BROKER_PASSWORD_FILE: /run/secrets/broker_password + BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1} + JAVA_TOOL_OPTIONS: "-XX:+ExitOnOutOfMemoryError" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - db_password + - s3_api_access_key_id + - s3_api_secret_access_key + - broker_password + tmpfs: + - /tmp:size=32m,mode=1777 + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1g + restart: unless-stopped + lb: image: haproxy:3.2.4-alpine volumes: @@ -200,6 +283,8 @@ secrets: file: ./secrets/db_password redis_password: file: ./secrets/redis_password + broker_password: + file: ./secrets/broker_password s3_access_key_id: file: ./secrets/s3_access_key_id @@ -214,3 +299,4 @@ secrets: volumes: postgres-data: s3-data: + broker-data: diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 00000000..96df8612 --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1,6 @@ +Upload: 202 queued, then ready; instance=7a70d3b286da +Anonymous S3 GET of known object: 403 +Outsider metadata/content: 404 +Download: 200; instance=f000ea12f8bd +SHA-256: 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +PASS: uploading replica stopped; original served by another replica; stopped replica restored diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..060802d0 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,4 @@ +$ make check +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +Exit code: 0 diff --git a/evidence/compose.txt b/evidence/compose.txt new file mode 100644 index 00000000..c89ebc6b --- /dev/null +++ b/evidence/compose.txt @@ -0,0 +1,13 @@ +$ docker compose config --quiet +Exit code: 0 + +$ docker compose ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "java -XX:MaxRAMPerc…" api 8 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "java -XX:MaxRAMPerc…" api 8 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-messenger-lab5-broker-1 rabbitmq@sha256:cd4009416f83d7e5895020cd71b0963f3b8a4dfc3de74bbb6fb3bfe5f81f01e0 "/bin/sh /bootstrap.…" broker 19 hours ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-messenger-lab5-lb-1 haproxy:3.2.4-alpine "docker-entrypoint.s…" lb 19 hours ago Up About a minute (healthy) 127.0.0.1:8085->8080/tcp +mtusi-messenger-lab5-postgres-1 postgres:17.6-alpine3.22 "docker-entrypoint.s…" postgres 19 hours ago Up 2 minutes (healthy) 5432/tcp +mtusi-messenger-lab5-redis-1 redis:8.4.0-alpine "docker-entrypoint.s…" redis 19 hours ago Up 2 minutes (healthy) 6379/tcp +mtusi-messenger-lab5-s3-1 quay.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e "/usr/bin/docker-ent…" s3 19 hours ago Up 2 minutes (healthy) 9000/tcp +mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "java -XX:MaxRAMPerc…" worker 8 minutes ago Up 2 minutes (healthy) 8080/tcp diff --git a/evidence/gradle-tests.txt b/evidence/gradle-tests.txt new file mode 100644 index 00000000..56fa37a2 --- /dev/null +++ b/evidence/gradle-tests.txt @@ -0,0 +1,45 @@ +Command: rtk gradlew compileKotlin compileTestKotlin test --console=plain (JDK 21) +BUILD SUCCESSFUL +Tests: 94; failures: 0; errors: 0; skipped: 0 +com.mtusi.messenger.main.infrastructure.controllers.attachment.AttachmentControllerTest: tests=3 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.authentication.AuthenticationControllerTest: tests=9 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.chat.ChatControllerTest: tests=6 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.health.HealthControllerTest: tests=2 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.message.MessageConcurrencyTest: tests=1 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.message.MessageControllerTest: tests=7 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.message.MessageIdempotencyTest: tests=7 failures=0 +com.mtusi.messenger.main.infrastructure.controllers.user.UserControllerTest: tests=11 failures=0 +com.mtusi.messenger.main.infrastructure.dto.AttachmentJobContractTest: tests=1 failures=0 + job envelope requires every field mandated by the public schema() +com.mtusi.messenger.main.infrastructure.repositories.authentication.session.cache.RedisSessionCacheRepositoryTest: tests=3 failures=0 +com.mtusi.messenger.main.infrastructure.repositories.chat.ExposedChatRepositoryTest: tests=2 failures=0 +com.mtusi.messenger.main.infrastructure.repositories.message.MessageAttachmentsQueryTest: tests=1 failures=0 +com.mtusi.messenger.main.infrastructure.services.attachment.AttachmentCleanupTest: tests=2 failures=0 + expired failed attachments lose partially published files() + orphan cleanup respects grace and fences late publishers() +com.mtusi.messenger.main.infrastructure.services.attachment.AttachmentPipelineTest: tests=4 failures=0 + attempts are bounded and failed attachments reject content and message links() + busy redelivery lets the next attachment finish before lease expiration() + worker distinguishes permanent image errors from bounded transient failures() + stopped worker leaves a durable queued attachment and redelivery preserves its result() +com.mtusi.messenger.main.infrastructure.services.attachment.ImageProcessingServiceTest: tests=6 failures=0 + PNG EXIF orientation is applied before resizing() + transparent and semitransparent pixels are composited on white() + invalid images and MIME mismatches are rejected before publishing a thumbnail() + missing or unknown EXIF orientation preserves the normal image() + all JPEG EXIF orientations rotate or mirror pixels and strip orientation from the thumbnail() + thumbnail fits the box floors dimensions and never enlarges the source() +com.mtusi.messenger.main.infrastructure.services.attachment.ImageValidatorTest: tests=3 failures=0 +com.mtusi.messenger.main.infrastructure.services.attachment.LegacyAttachmentMigrationTest: tests=1 failures=0 + migration schedules a thumbnail for an existing lab4 original() +com.mtusi.messenger.main.infrastructure.services.attachment.jobs.RabbitMqJobBrokerServiceTest: tests=1 failures=0 + confirmed job survives broker restart and negative acknowledgement preserves its identity() +com.mtusi.messenger.main.infrastructure.services.authentication.CrossReplicaRevocationTest: tests=2 failures=0 +com.mtusi.messenger.main.infrastructure.services.authentication.SessionLookupSingleFlightTest: tests=4 failures=0 +com.mtusi.messenger.main.infrastructure.services.hashing.DefaultHashingServiceTest: tests=4 failures=0 +com.mtusi.messenger.main.plugins.CorsTest: tests=2 failures=0 +com.mtusi.messenger.main.plugins.DatabaseMigrationTest: tests=1 failures=0 +com.mtusi.messenger.main.plugins.DependencyFailureTest: tests=5 failures=0 +com.mtusi.messenger.main.plugins.InstanceIdTest: tests=1 failures=0 +com.mtusi.messenger.main.plugins.SessionCacheTest: tests=3 failures=0 +com.mtusi.messenger.main.utils.TemporaryUploadTest: tests=2 failures=0 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..a380f1d9 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,23 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + + +Before restart: +/mtusi-messenger-lab5-api-1 2026-11-17T11:48:25.6974278Z +/mtusi-messenger-lab5-api-2 2026-11-17T11:48:25.799164633Z +/mtusi-messenger-lab5-broker-1 2026-11-17T11:48:15.954373295Z +/mtusi-messenger-lab5-lb-1 2026-11-17T11:48:28.901775676Z +/mtusi-messenger-lab5-postgres-1 2026-11-17T11:48:15.943166503Z +/mtusi-messenger-lab5-redis-1 2026-11-17T11:48:15.95361167Z +/mtusi-messenger-lab5-s3-1 2026-11-17T11:48:15.94897392Z +/mtusi-messenger-lab5-worker-1 2026-11-17T11:48:25.696508175Z +After restart: +/mtusi-messenger-lab5-api-1 2026-11-17T11:52:14.708903753Z +/mtusi-messenger-lab5-api-2 2026-11-17T11:52:14.820903586Z +/mtusi-messenger-lab5-broker-1 2026-11-17T11:52:04.990389887Z +/mtusi-messenger-lab5-lb-1 2026-11-17T11:52:17.919454796Z +/mtusi-messenger-lab5-postgres-1 2026-11-17T11:52:04.990403929Z +/mtusi-messenger-lab5-redis-1 2026-11-17T11:52:04.984496471Z +/mtusi-messenger-lab5-s3-1 2026-11-17T11:52:04.992642512Z +/mtusi-messenger-lab5-worker-1 2026-11-17T11:52:14.708902794Z diff --git a/evidence/s3-permissions.txt b/evidence/s3-permissions.txt new file mode 100644 index 00000000..59841603 --- /dev/null +++ b/evidence/s3-permissions.txt @@ -0,0 +1,10 @@ + Container mtusi-messenger-lab5-s3-1 Running +Bucket created successfully `local/messenger-attachments`. +Access permission for `local/messenger-attachments` is set to `private` + Container mtusi-messenger-lab5-s3-1 Running +Bucket created successfully `local/messenger-attachments`. +Access permission for `local/messenger-attachments` is set to `private` + Container mtusi-messenger-lab5-s3-1 Running +PASS: app cannot administer MinIO, create buckets, change policy or write outside originals/ +wget: can't connect to remote host (172.26.0.2): Connection refused +PASS: console unavailable from shared network diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..2448312f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/worker-recovery.txt b/evidence/worker-recovery.txt new file mode 100644 index 00000000..bcac0aeb --- /dev/null +++ b/evidence/worker-recovery.txt @@ -0,0 +1,3 @@ +PASS: worker stopped; upload 202 queued; message accepted; content 409; attachment_id=0305bfe5-e30e-47cc-b81e-4f2c578273a6 +PASS: both APIs and broker restarted with existing volumes; same attachment remains queued; attachment_id=0305bfe5-e30e-47cc-b81e-4f2c578273a6 +PASS: same accepted job reached ready; original unchanged; thumbnail JPEG 256x160 with matching checksum; attachment_id=0305bfe5-e30e-47cc-b81e-4f2c578273a6 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ec020b4d..b51abcf0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,6 +16,9 @@ lettuce = "6.7.1.RELEASE" bouncycastle = "1.81" test-containers = "1.21.0" +rabbitmq = "5.26.0" +metadata-extractor = "2.19.0" + [plugins] kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } @@ -52,3 +55,6 @@ bouncycastle = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "boun testContainers = { module = "org.testcontainers:testcontainers", version.ref = "test-containers" } testContainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "test-containers" } + +rabbitmq = { module = "com.rabbitmq:amqp-client", version.ref = "rabbitmq" } +metadata-extractor = { module = "com.drewnoakes:metadata-extractor", version.ref = "metadata-extractor" } diff --git a/infra/rabbitmq/bootstrap.sh b/infra/rabbitmq/bootstrap.sh new file mode 100644 index 00000000..2796c6c6 --- /dev/null +++ b/infra/rabbitmq/bootstrap.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +# The password is read only inside the container and is never written to logs. +export RABBITMQ_DEFAULT_USER=messenger +export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)" +exec docker-entrypoint.sh rabbitmq-server diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 00000000..0c013283 --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,3 @@ +# Job envelopes contain identifiers only, never image bytes. +max_message_size = 16384 +consumer_timeout = 300000 diff --git a/infra/s3/bootstrap.sh b/infra/s3/bootstrap.sh index 5ca311de..45dd2296 100644 --- a/infra/s3/bootstrap.sh +++ b/infra/s3/bootstrap.sh @@ -12,7 +12,7 @@ cat > /tmp/attachment-policy.json < "secrets/$name" echo "created secrets/$name" diff --git a/scripts/restart.sh b/scripts/restart.sh index 88dbc5d9..3b64ec43 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,7 @@ #!/bin/sh -# Restarts the balancer, every API replica, PostgreSQL, Redis and S3, keeping the named volume (no `down -v`, no re-seed). +# Restarts the balancer, every API replica, PostgreSQL, Redis, S3 and broker, keeping the named volume (no `down -v`, no re-seed). set -eu cd "$(dirname "$0")/.." -docker compose stop lb api redis postgres s3 +docker compose stop lb api worker redis postgres s3 broker docker compose up -d --wait diff --git a/src/main/kotlin/com/mtusi/messenger/main/App.kt b/src/main/kotlin/com/mtusi/messenger/main/App.kt index d2214cd9..97e9e9a9 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/App.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/App.kt @@ -9,21 +9,35 @@ import com.mtusi.messenger.main.plugins.configureDatabase import com.mtusi.messenger.main.plugins.configureDefaultHeaders import com.mtusi.messenger.main.plugins.configureGracefulShutdown import com.mtusi.messenger.main.plugins.configureInstanceId +import com.mtusi.messenger.main.plugins.configureJobBroker import com.mtusi.messenger.main.plugins.configureKoin import com.mtusi.messenger.main.plugins.configureRedis import com.mtusi.messenger.main.plugins.configureRequestId -import com.mtusi.messenger.main.plugins.configureSessionRevocationSync import com.mtusi.messenger.main.plugins.configureRouting +import com.mtusi.messenger.main.plugins.configureSessionRevocationSync import com.mtusi.messenger.main.plugins.configureStatusPages import com.mtusi.messenger.main.plugins.migrateDatabase import io.ktor.server.application.Application import io.ktor.server.netty.EngineMain fun main(args: Array) { + + val command = args.firstOrNull() + // `migrate` applies database migrations and exits; the API replicas start with migrations already in place. - if (args.firstOrNull() == "migrate") { + if (command == "migrate") { return migrateDatabase() } + + if (command == "worker") { + val workerArgs = arrayOf( + "-config=worker.conf", + *args.drop(1).toTypedArray() + ) + + return EngineMain.main(workerArgs) + } + EngineMain.main(args) } @@ -31,6 +45,7 @@ fun Application.module() { configureKoin() configureDatabase() + configureJobBroker() configureRedis() configureGracefulShutdown() configureSessionRevocationSync() diff --git a/src/main/kotlin/com/mtusi/messenger/main/config/broker/BrokerConfig.kt b/src/main/kotlin/com/mtusi/messenger/main/config/broker/BrokerConfig.kt new file mode 100644 index 00000000..e5c1acc8 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/config/broker/BrokerConfig.kt @@ -0,0 +1,16 @@ +package com.mtusi.messenger.main.config.broker + +interface BrokerConfig { + + val host: String + val port: Int + val queue: String + + val username: String + val password: String? + val passwordFile: String? + + val connectionTimeoutMs: Int + val operationTimeoutMs: Int + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/config/broker/DefaultBrokerConfig.kt b/src/main/kotlin/com/mtusi/messenger/main/config/broker/DefaultBrokerConfig.kt new file mode 100644 index 00000000..22b157d7 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/config/broker/DefaultBrokerConfig.kt @@ -0,0 +1,16 @@ +package com.mtusi.messenger.main.config.broker + +import io.ktor.server.application.Application +import io.ktor.server.config.getAs +import org.koin.core.annotation.Provided +import org.koin.core.annotation.Single + +@Single +class DefaultBrokerConfig( + @Provided app: Application +) : BrokerConfig by ( + app.environment.config + .propertyOrNull("broker") + ?.getAs() + ?: SerialBrokerConfig() +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/config/broker/SerialBrokerConfig.kt b/src/main/kotlin/com/mtusi/messenger/main/config/broker/SerialBrokerConfig.kt new file mode 100644 index 00000000..3d66b573 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/config/broker/SerialBrokerConfig.kt @@ -0,0 +1,15 @@ +package com.mtusi.messenger.main.config.broker + +import kotlinx.serialization.Serializable + +@Serializable +data class SerialBrokerConfig( + override val host: String = "localhost", + override val port: Int = 5672, + override val username: String = "messenger", + override val password: String? = null, + override val passwordFile: String? = null, + override val queue: String = "attachment.process.v1", + override val connectionTimeoutMs: Int = 3000, + override val operationTimeoutMs: Int = 8000 +) : BrokerConfig diff --git a/src/main/kotlin/com/mtusi/messenger/main/config/worker/DefaultWorkerConfig.kt b/src/main/kotlin/com/mtusi/messenger/main/config/worker/DefaultWorkerConfig.kt new file mode 100644 index 00000000..f32fd270 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/config/worker/DefaultWorkerConfig.kt @@ -0,0 +1,16 @@ +package com.mtusi.messenger.main.config.worker + +import io.ktor.server.application.Application +import io.ktor.server.config.getAs +import org.koin.core.annotation.Provided +import org.koin.core.annotation.Single + +@Single +class DefaultWorkerConfig( + @Provided app: Application +) : WorkerConfig by ( + app.environment.config + .propertyOrNull("worker") + ?.getAs() + ?: SerialWorkerConfig() +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/config/worker/SerialWorkerConfig.kt b/src/main/kotlin/com/mtusi/messenger/main/config/worker/SerialWorkerConfig.kt new file mode 100644 index 00000000..7aad65f3 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/config/worker/SerialWorkerConfig.kt @@ -0,0 +1,20 @@ +package com.mtusi.messenger.main.config.worker + +import kotlinx.serialization.Serializable + +@Serializable +data class SerialWorkerConfig( + override val concurrency: Int = 1, + override val maxAttempts: Int = 5, + override val timeoutSeconds: Long = 60, + override val stagingRetentionSeconds: Long = 86400 +) : WorkerConfig { + + init { + require(concurrency in 1..4) + require(maxAttempts in 1..100) + require(timeoutSeconds in 1..3600) + require(stagingRetentionSeconds >= 3600) + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/config/worker/WorkerConfig.kt b/src/main/kotlin/com/mtusi/messenger/main/config/worker/WorkerConfig.kt new file mode 100644 index 00000000..f5c31ad8 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/config/worker/WorkerConfig.kt @@ -0,0 +1,10 @@ +package com.mtusi.messenger.main.config.worker + +interface WorkerConfig { + + val concurrency: Int + val maxAttempts: Int + val timeoutSeconds: Long + val stagingRetentionSeconds: Long + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentController.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentController.kt index 6ae1f5a8..556094e2 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentController.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentController.kt @@ -7,19 +7,28 @@ import com.mtusi.messenger.main.infrastructure.types.ApiException import com.mtusi.messenger.main.infrastructure.types.ErrorCode import com.mtusi.messenger.main.plugins.authentication.SESSION_AUTH import com.mtusi.messenger.main.utils.requireUuid -import io.ktor.http.* -import io.ktor.server.auth.authenticate -import io.ktor.server.request.* -import io.ktor.server.response.* -import io.ktor.server.routing.* -import io.ktor.utils.io.readAvailable -import kotlinx.coroutines.withTimeoutOrNull -import org.koin.ktor.ext.inject import com.mtusi.messenger.main.utils.withTemporaryUpload +import io.ktor.http.BadContentTypeFormatException +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.auth.authenticate +import io.ktor.server.request.contentLength +import io.ktor.server.request.contentType +import io.ktor.server.request.receiveChannel +import io.ktor.server.response.header +import io.ktor.server.response.respond +import io.ktor.server.response.respondBytes +import io.ktor.server.routing.Routing +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.utils.io.readAvailable import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import org.koin.ktor.ext.inject import kotlin.time.Duration.Companion.milliseconds fun Routing.configureAttachmentRouting() { @@ -34,20 +43,32 @@ fun Routing.configureAttachmentRouting() { val actor = call.currentUser val chatId = requireUuid(call.parameters["chat_id"], "chat_id") - service.requireMember(chatId, actor.id) + service.requireMember( + chatId = chatId, + actorId = actor.id + ) val filename = call.request.queryParameters["filename"] ?: throw ApiException(ErrorCode.INVALID_REQUEST, "filename is required") - if (filename.isBlank() || filename.codePointCount(0, filename.length) !in 1..255 || filename.any { it == '/' || it == '\\' || it.isISOControl() }) { + if ( + filename.isBlank() || + filename.codePointCount(0, filename.length) !in 1..255 || + filename.any { it == '/' || it == '\\' || it.isISOControl() } + ) { throw ApiException(ErrorCode.INVALID_REQUEST, "invalid filename") } val type = try { - call.request.contentType().withoutParameters().toString().lowercase() + call.request + .contentType() + .withoutParameters() + .toString() + .lowercase() } catch (_: BadContentTypeFormatException) { throw ApiException(ErrorCode.INVALID_REQUEST, "invalid Content-Type") } + if (type !in setOf("image/png", "image/jpeg")) { throw ApiException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "expected PNG or JPEG") } @@ -63,44 +84,89 @@ fun Routing.configureAttachmentRouting() { val input = call.receiveChannel() val buffer = ByteArray(8192) var size = 0L + file.outputStream().use { output -> while (true) { val count = input.readAvailable(buffer) - if (count == -1) break + + if (count == -1) { + break + } + size += count + if (size > MAX_ATTACHMENT_BYTES) { throw ApiException(ErrorCode.PAYLOAD_TOO_LARGE, "file exceeds 10 MiB") } + output.write(buffer, 0, count) } } } } ?: throw ApiException(ErrorCode.INVALID_REQUEST, "upload timed out") - val attachment = service.upload(chatId, actor.id, filename, type, file) - call.respond(HttpStatusCode.Created, attachment) + val attachment = service.upload( + chatId = chatId, + actorId = actor.id, + filename = filename, + contentType = type, + file = file + ) + + call.respond(HttpStatusCode.Accepted, attachment) } } } get("/api/v1/attachments/{id}") { - call.respond(service.find(requireUuid(call.parameters["id"], "id"), call.currentUser.id)) + + val id = requireUuid(call.parameters["id"], "id") + val attachment = service.find( + id = id, + actorId = call.currentUser.id + ) + + call.respond(attachment) } get("/api/v1/attachments/{id}/content") { - val attachment = service.find(requireUuid(call.parameters["id"], "id"), call.currentUser.id) + val id = requireUuid(call.parameters["id"], "id") + val attachment = service.find( + id = id, + actorId = call.currentUser.id + ) - if ((call.request.queryParameters["variant"] ?: "original") != "original") { - throw ApiException(ErrorCode.NOT_FOUND, "variant not found") + val variant = call.request.queryParameters["variant"] ?: "original" + + if (attachment.status != "ready") { + throw ApiException(ErrorCode.CONFLICT, "attachment is not ready") + } + + val contentType = if (variant == "original") { + attachment.contentType + } else { + attachment.variants + .firstOrNull { it.name == variant } + ?.contentType + ?: throw ApiException(ErrorCode.NOT_FOUND, "variant not found") } call.response.header(HttpHeaders.CacheControl, "private, no-store") call.response.header("X-Content-Type-Options", "nosniff") - call.respondBytes(service.content(attachment), ContentType.parse(attachment.contentType)) + val bytes = service.content( + attachment = attachment, + variant = variant + ) + + call.respondBytes( + bytes = bytes, + contentType = ContentType.parse(contentType) + ) } } + } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt index d17fd116..5ebfb74e 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/controllers/health/HealthController.kt @@ -2,13 +2,11 @@ package com.mtusi.messenger.main.infrastructure.controllers.health import com.mtusi.messenger.main.extensions.respondError import com.mtusi.messenger.main.infrastructure.controllers.health.responses.HealthResponse +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ErrorCode import com.mtusi.messenger.main.plugins.isDraining import com.mtusi.messenger.main.plugins.suspendedTransaction -import com.mtusi.messenger.main.infrastructure.repositories.attachment.storage.ObjectStorageRepository -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import org.koin.ktor.ext.inject import io.ktor.http.HttpStatusCode import io.ktor.server.application.log import io.ktor.server.response.respond @@ -16,8 +14,11 @@ import io.ktor.server.routing.Routing import io.ktor.server.routing.get import io.ktor.server.routing.route import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.withContext +import org.koin.ktor.ext.inject fun Routing.configureHealthRouting() = route("/health") { @@ -25,9 +26,10 @@ fun Routing.configureHealthRouting() = route("/health") { call.respond(HttpStatusCode.OK, HealthResponse()) } - val storage by inject() + val storage by inject() + val broker by inject() - // PostgreSQL and S3 are dependencies of the synchronous path; Redis has a DB fallback. + // Readiness checks PostgreSQL, S3 and RabbitMQ. Redis has a DB fallback. get("ready") { // SIGTERM received: leave the balancer's rotation while requests in flight are still being finished. @@ -40,6 +42,7 @@ fun Routing.configureHealthRouting() = route("/health") { suspendedTransaction { exec("SELECT 1") } + true } catch (cause: Exception) { @@ -48,6 +51,7 @@ fun Routing.configureHealthRouting() = route("/health") { } call.application.log.warn("Readiness check failed: ${cause.message}") + false } @@ -56,11 +60,20 @@ fun Routing.configureHealthRouting() = route("/health") { return@get } - if (!withContext(Dispatchers.IO) { storage.ready() }) { + val isStorageReachable = withContext(Dispatchers.IO) { + storage.ready() + } + + if (!isStorageReachable) { call.respondError(ErrorCode.UNAVAILABLE, "object storage unavailable") return@get } + if (!broker.ready()) { + call.respondError(ErrorCode.UNAVAILABLE, "job broker unavailable") + return@get + } + call.respond(HttpStatusCode.OK, HealthResponse()) } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/Attachment.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/Attachment.kt index e65ac4cd..f2aeccf6 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/Attachment.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/Attachment.kt @@ -16,6 +16,6 @@ data class Attachment( val createdAt: Instant, val updatedAt: Instant = createdAt, val status: String = "ready", - val variants: List = emptyList(), + val variants: List = emptyList(), val errorCode: String? = null, ) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentJob.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentJob.kt new file mode 100644 index 00000000..08a39ede --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentJob.kt @@ -0,0 +1,25 @@ +package com.mtusi.messenger.main.infrastructure.dto + +import com.mtusi.messenger.main.compat.serializers.SerialUUID +import kotlinx.serialization.Required +import kotlinx.serialization.Serializable +import kotlin.time.Instant + +@Serializable +data class AttachmentJob( + val jobId: SerialUUID, + val attachmentId: SerialUUID, + val createdAt: Instant, + + @Required + val schemaVersion: Int = 1, + + @Required + val type: String = "attachment.process.v1", + + @Required + val pipelineVersion: Int = 1, + + @Required + val attempt: Int = 1, +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentVariant.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentVariant.kt new file mode 100644 index 00000000..7b65ef86 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentVariant.kt @@ -0,0 +1,13 @@ +package com.mtusi.messenger.main.infrastructure.dto + +import kotlinx.serialization.Serializable + +@Serializable +data class AttachmentVariant( + val name: String, + val contentType: String, + val sizeBytes: Long, + val width: Int, + val height: Int, + val sha256: String, +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/ProcessedImage.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/ProcessedImage.kt new file mode 100644 index 00000000..dcaddab8 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/ProcessedImage.kt @@ -0,0 +1,39 @@ +package com.mtusi.messenger.main.infrastructure.dto + +data class ProcessedImage( + val bytes: ByteArray, + val width: Int, + val height: Int, + val contentType: String = "image/jpeg" +) { + + override fun equals( + other: Any? + ): Boolean { + + if (this === other) { + return true + } + + if (javaClass != other?.javaClass) { + return false + } + + other as ProcessedImage + + return width == other.width && + height == other.height && + bytes.contentEquals(other.bytes) && + contentType == other.contentType + } + + override fun hashCode(): Int { + var result = width + result = 31 * result + height + result = 31 * result + bytes.contentHashCode() + result = 31 * result + contentType.hashCode() + + return result + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt new file mode 100644 index 00000000..2e64b26d --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentJobModel.kt @@ -0,0 +1,24 @@ +package com.mtusi.messenger.main.infrastructure.models.attachment + +import org.jetbrains.exposed.v1.core.dao.id.java.UUIDTable +import org.jetbrains.exposed.v1.core.java.javaUUID +import org.jetbrains.exposed.v1.datetime.timestamp + +object AttachmentJobModel : UUIDTable("attachment_jobs") { + + val attachmentId = reference( + name = "attachment_id", + foreign = AttachmentModel + ).uniqueIndex() + + val pipelineVersion = integer("pipeline_version").default(1) + val attempts = integer("attempts").default(0) + val leaseToken = javaUUID("lease_token").nullable() + val leaseUntil = timestamp("lease_until").nullable() + val lastErrorCode = varchar("last_error_code", 64).nullable() + val nextAttemptAt = timestamp("next_attempt_at").nullable() + val createdAt = timestamp("created_at") + val legacyOriginal = bool("legacy_original").default(false) + val legacyDispatchPending = bool("legacy_dispatch_pending").default(false) + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt index 2a26edd0..027f3b7e 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentModel.kt @@ -13,4 +13,8 @@ object AttachmentModel : UUIDTable("attachments") { val sizeBytes = long("size_bytes") val sha256 = varchar("sha256", 64) val createdAt = timestamp("created_at") + val updatedAt = timestamp("updated_at") + val status = varchar("status", 16).default("ready") + val stagingCleaned = bool("staging_cleaned").default(false) + val errorCode = varchar("error_code", 64).nullable() } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentVariantModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentVariantModel.kt new file mode 100644 index 00000000..0bcc0479 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/attachment/AttachmentVariantModel.kt @@ -0,0 +1,23 @@ +package com.mtusi.messenger.main.infrastructure.models.attachment + +import org.jetbrains.exposed.v1.core.ReferenceOption +import org.jetbrains.exposed.v1.core.Table + +object AttachmentVariantModel : Table("attachment_variants") { + + val attachmentId = reference( + name = "attachment_id", + foreign = AttachmentModel, + onDelete = ReferenceOption.CASCADE + ) + + val name = varchar("name", 16) + val contentType = varchar("content_type", 32) + val sizeBytes = long("size_bytes") + val width = integer("width") + val height = integer("height") + val sha256 = varchar("sha256", 64) + + override val primaryKey = PrimaryKey(attachmentId, name) + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt index 1487d761..fd4086db 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/AttachmentRepository.kt @@ -1,7 +1,9 @@ package com.mtusi.messenger.main.infrastructure.repositories.attachment import com.mtusi.messenger.main.infrastructure.dto.Attachment +import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant import java.util.UUID +import kotlin.time.Instant interface AttachmentRepository { @@ -9,6 +11,35 @@ interface AttachmentRepository { fun findById(id: UUID): Attachment? + fun findStatusesInChat( + chatId: UUID, + ids: List, + lockForUpdate: Boolean = false + ): List + + fun setState( + id: UUID, + status: String, + errorCode: String? = null + ) + + fun markEnqueueFailed( + id: UUID + ) + + fun markStagingCleaned( + id: UUID + ) + + fun findExpiredStorage( + before: Instant + ): List + + fun publish( + id: UUID, + variant: AttachmentVariant + ) + fun attach( messageId: UUID, ids: List @@ -18,3 +49,9 @@ interface AttachmentRepository { fun idsForMessages(messageIds: List): Map> } + +data class ExpiredAttachmentStorage( + val id: UUID, + val failed: Boolean, + val legacyOriginal: Boolean +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt index 024c6330..f1144526 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/ExposedAttachmentRepository.kt @@ -1,21 +1,34 @@ package com.mtusi.messenger.main.infrastructure.repositories.attachment import com.mtusi.messenger.main.infrastructure.dto.Attachment +import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentVariantModel import com.mtusi.messenger.main.infrastructure.models.attachment.MessageAttachmentModel -import org.jetbrains.exposed.v1.jdbc.insert -import org.jetbrains.exposed.v1.jdbc.selectAll +import com.mtusi.messenger.main.utils.nowMicros import org.jetbrains.exposed.v1.core.ResultRow -import org.jetbrains.exposed.v1.core.inList -import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.less +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update import org.koin.core.annotation.Single import java.util.UUID +import kotlin.time.Clock +import kotlin.time.Instant @Single class ExposedAttachmentRepository : AttachmentRepository { - override fun create(attachment: Attachment) { + override fun create( + attachment: Attachment + ) { AttachmentModel.insert { it[id] = attachment.id it[chatId] = attachment.chatId @@ -25,17 +38,129 @@ class ExposedAttachmentRepository : AttachmentRepository { it[sizeBytes] = attachment.sizeBytes it[sha256] = attachment.sha256 it[createdAt] = attachment.createdAt + it[updatedAt] = attachment.updatedAt + it[status] = attachment.status + it[errorCode] = attachment.errorCode } } override fun findById( id: UUID ): Attachment? { - return AttachmentModel + + val rows = AttachmentModel + .leftJoin(AttachmentVariantModel) .selectAll() .where(AttachmentModel.id eq id) - .firstOrNull() - ?.toAttachment() + .orderBy(AttachmentVariantModel.name to SortOrder.ASC) + .toList() + + val attachment = rows.firstOrNull() + ?: return null + + val variants = rows.mapNotNull { it.toAttachmentVariantOrNull() } + + return attachment.toAttachment(variants) + } + + override fun findStatusesInChat( + chatId: UUID, + ids: List, + lockForUpdate: Boolean + ): List { + + if (ids.isEmpty()) { + return emptyList() + } + + val query = AttachmentModel + .select(AttachmentModel.status) + .where((AttachmentModel.chatId eq chatId) and (AttachmentModel.id inList ids)) + + if (lockForUpdate) { + // Requests with overlapping attachments take locks in the same order. + query + .orderBy(AttachmentModel.id to SortOrder.ASC) + .forUpdate() + } + + return query.map { it[AttachmentModel.status] } + } + + override fun setState( + id: UUID, + status: String, + errorCode: String? + ) { + AttachmentModel.update(where = { AttachmentModel.id eq id }) { + it[this.status] = status + it[this.errorCode] = errorCode + it[updatedAt] = Clock.System.nowMicros() + } + } + + override fun markEnqueueFailed( + id: UUID + ) { + AttachmentModel.update( + where = { + (AttachmentModel.id eq id) and (AttachmentModel.status eq "queued") + } + ) { + it[status] = "failed" + it[errorCode] = "enqueue_failed" + it[updatedAt] = Clock.System.nowMicros() + } + } + + override fun markStagingCleaned( + id: UUID + ) { + AttachmentModel.update(where = { AttachmentModel.id eq id }) { + it[stagingCleaned] = true + } + } + + override fun findExpiredStorage( + before: Instant + ): List { + return AttachmentModel + .leftJoin(AttachmentJobModel) + .select(AttachmentModel.id, AttachmentModel.status, AttachmentJobModel.legacyOriginal) + .where( + (AttachmentModel.status inList listOf("ready", "failed")) + .and(AttachmentModel.updatedAt less before) + .and(AttachmentModel.stagingCleaned eq false) + ) + .orderBy(AttachmentModel.updatedAt to SortOrder.ASC) + .limit(100) + .map { + ExpiredAttachmentStorage( + id = it[AttachmentModel.id].value, + failed = it[AttachmentModel.status] == "failed", + legacyOriginal = it.getOrNull(AttachmentJobModel.legacyOriginal) == true + ) + } + } + + override fun publish( + id: UUID, + variant: AttachmentVariant + ) { + AttachmentVariantModel.insertIgnore { + it[attachmentId] = id + it[name] = variant.name + it[contentType] = variant.contentType + it[sizeBytes] = variant.sizeBytes + it[width] = variant.width + it[height] = variant.height + it[sha256] = variant.sha256 + } + + setState( + id = id, + status = "ready" + ) } override fun attach( @@ -74,7 +199,9 @@ class ExposedAttachmentRepository : AttachmentRepository { ) } - private fun ResultRow.toAttachment() = Attachment( + private fun ResultRow.toAttachment( + variants: List + ) = Attachment( id = this[AttachmentModel.id].value, chatId = this[AttachmentModel.chatId].value, ownerId = this[AttachmentModel.ownerId].value, @@ -82,7 +209,26 @@ class ExposedAttachmentRepository : AttachmentRepository { contentType = this[AttachmentModel.contentType], sizeBytes = this[AttachmentModel.sizeBytes], sha256 = this[AttachmentModel.sha256], - createdAt = this[AttachmentModel.createdAt] + createdAt = this[AttachmentModel.createdAt], + updatedAt = this[AttachmentModel.updatedAt], + status = this[AttachmentModel.status], + errorCode = this[AttachmentModel.errorCode], + variants = variants ) + private fun ResultRow.toAttachmentVariantOrNull(): AttachmentVariant? { + + val name = getOrNull(AttachmentVariantModel.name) + ?: return null + + return AttachmentVariant( + name = name, + contentType = this[AttachmentVariantModel.contentType], + sizeBytes = this[AttachmentVariantModel.sizeBytes], + width = this[AttachmentVariantModel.width], + height = this[AttachmentVariantModel.height], + sha256 = this[AttachmentVariantModel.sha256] + ) + } + } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt new file mode 100644 index 00000000..270d47e2 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/AttachmentJobRepository.kt @@ -0,0 +1,59 @@ +package com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant +import java.util.UUID + +interface AttachmentJobRepository { + + fun create( + job: AttachmentJob + ) + + fun claim( + job: AttachmentJob, + token: UUID, + maxAttempts: Int, + leaseSeconds: Long + ): JobClaim + + fun complete( + job: AttachmentJob, + token: UUID, + variant: AttachmentVariant + ): Boolean + + fun fail( + job: AttachmentJob, + token: UUID, + errorCode: String, + permanent: Boolean, + maxAttempts: Int + ): Boolean + + fun findPendingLegacyJobs( + limit: Int + ): List + + fun markLegacyJobDispatched( + id: UUID + ) + +} + +sealed interface JobClaim { + + data object Complete : JobClaim + + data object Busy : JobClaim + + data class Retry( + val attempt: Int + ) : JobClaim + + data class Acquired( + val attempt: Int, + val legacyOriginal: Boolean = false + ) : JobClaim + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt new file mode 100644 index 00000000..7de5e212 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/jobs/ExposedAttachmentJobRepository.kt @@ -0,0 +1,250 @@ +package com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel +import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository +import com.mtusi.messenger.main.utils.exposed.operations.plus +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.greater +import org.jetbrains.exposed.v1.core.isNotNull +import org.jetbrains.exposed.v1.core.longLiteral +import org.jetbrains.exposed.v1.datetime.CurrentTimestamp +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import org.koin.core.annotation.Single +import java.util.UUID + +@Single +class ExposedAttachmentJobRepository( + private val attachmentRepository: AttachmentRepository +) : AttachmentJobRepository { + + override fun create( + job: AttachmentJob + ) { + AttachmentJobModel.insert { + it[id] = job.jobId + it[attachmentId] = job.attachmentId + it[pipelineVersion] = job.pipelineVersion + it[createdAt] = job.createdAt + } + } + + override fun claim( + job: AttachmentJob, + token: UUID, + maxAttempts: Int, + leaseSeconds: Long + ): JobClaim { + + val row = lock(job) ?: return JobClaim.Complete + + val status = AttachmentModel + .select(AttachmentModel.status) + .where(AttachmentModel.id eq job.attachmentId) + .forUpdate() + .single()[AttachmentModel.status] + + if (status in setOf("ready", "failed")) { + return JobClaim.Complete + } + + val activeLease = AttachmentJobModel.leaseUntil.isNotNull() + .and(AttachmentJobModel.leaseUntil greater CurrentTimestamp) + + val busy = AttachmentJobModel + .select(activeLease) + .where(AttachmentJobModel.id eq job.jobId) + .single() + .get(activeLease) == true + + if (busy) { + return JobClaim.Busy + } + + val attempts = row[AttachmentJobModel.attempts] + + if (attempts >= maxAttempts) { + attachmentRepository.setState( + id = job.attachmentId, + status = "failed", + errorCode = "retry_exhausted" + ) + + clearLease(job.jobId) + + return JobClaim.Complete + } + + val nextAttempt = attempts + 1 + + if (job.attempt != nextAttempt) { + if (row[AttachmentJobModel.leaseToken] != null) { + queueRetry( + id = job.jobId, + attachmentId = job.attachmentId, + errorCode = "lease_expired" + ) + } + + return JobClaim.Retry(attempt = nextAttempt) + } + + AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { + it[this.attempts] = nextAttempt + it[leaseToken] = token + it[leaseUntil] = CurrentTimestamp + longLiteral(leaseSeconds) + it[nextAttemptAt] = null + } + + attachmentRepository.setState( + id = job.attachmentId, + status = "processing" + ) + + return JobClaim.Acquired( + attempt = nextAttempt, + legacyOriginal = row[AttachmentJobModel.legacyOriginal] + ) + } + + override fun complete( + job: AttachmentJob, + token: UUID, + variant: AttachmentVariant + ): Boolean { + + val row = lock(job) ?: return false + + if (row[AttachmentJobModel.leaseToken] != token) { + return false + } + + attachmentRepository.publish( + id = job.attachmentId, + variant = variant + ) + + clearLease(job.jobId) + + return true + } + + override fun fail( + job: AttachmentJob, + token: UUID, + errorCode: String, + permanent: Boolean, + maxAttempts: Int + ): Boolean { + + val row = lock(job) ?: return true + + if (row[AttachmentJobModel.leaseToken] != token) { + return false + } + + val terminal = permanent || row[AttachmentJobModel.attempts] >= maxAttempts + + if (!terminal) { + queueRetry( + id = job.jobId, + attachmentId = job.attachmentId, + errorCode = errorCode + ) + + return false + } + + attachmentRepository.setState( + id = job.attachmentId, + status = "failed", + errorCode = if (permanent) errorCode else "retry_exhausted" + ) + + AttachmentJobModel.update(where = { AttachmentJobModel.id eq job.jobId }) { + it[leaseToken] = null + it[leaseUntil] = null + it[lastErrorCode] = errorCode + it[nextAttemptAt] = null + } + + return true + } + + private fun queueRetry( + id: UUID, + attachmentId: UUID, + errorCode: String + ) { + attachmentRepository.setState( + id = attachmentId, + status = "queued" + ) + + AttachmentJobModel.update(where = { AttachmentJobModel.id eq id }) { + it[leaseToken] = null + it[leaseUntil] = null + it[lastErrorCode] = errorCode + it[nextAttemptAt] = CurrentTimestamp + } + } + + override fun findPendingLegacyJobs( + limit: Int + ): List { + return AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.legacyDispatchPending eq true) + .orderBy(AttachmentJobModel.createdAt to SortOrder.ASC, AttachmentJobModel.id to SortOrder.ASC) + .limit(limit) + .map { + AttachmentJob( + jobId = it[AttachmentJobModel.id].value, + attachmentId = it[AttachmentJobModel.attachmentId].value, + pipelineVersion = it[AttachmentJobModel.pipelineVersion], + createdAt = it[AttachmentJobModel.createdAt] + ) + } + } + + override fun markLegacyJobDispatched( + id: UUID + ) { + AttachmentJobModel.update(where = { AttachmentJobModel.id eq id }) { + it[legacyDispatchPending] = false + } + } + + private fun lock( + job: AttachmentJob + ): ResultRow? { + return AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.id eq job.jobId) + .forUpdate() + .firstOrNull() + ?.takeIf { + it[AttachmentJobModel.attachmentId].value == job.attachmentId && + it[AttachmentJobModel.pipelineVersion] == job.pipelineVersion + } + } + + private fun clearLease( + id: UUID + ) { + AttachmentJobModel.update(where = { AttachmentJobModel.id eq id }) { + it[leaseToken] = null + it[leaseUntil] = null + it[nextAttemptAt] = null + } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/storage/ObjectStorageRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/storage/ObjectStorageRepository.kt deleted file mode 100644 index 42154b81..00000000 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/storage/ObjectStorageRepository.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.mtusi.messenger.main.infrastructure.repositories.attachment.storage - -import java.io.File -import java.util.UUID - -interface ObjectStorageRepository { - - suspend fun put( - id: UUID, - file: File, - contentType: String - ) - - suspend fun get(id: UUID): ByteArray - - suspend fun delete(id: UUID) - - suspend fun ready(): Boolean -} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt index 125ac525..d7b41db8 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupService.kt @@ -1,16 +1,23 @@ package com.mtusi.messenger.main.infrastructure.services.attachment +import com.mtusi.messenger.main.config.worker.WorkerConfig +import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.UploadJournalRepository -import com.mtusi.messenger.main.infrastructure.repositories.attachment.storage.ObjectStorageRepository +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.plugins.suspendedTransaction import kotlinx.coroutines.CancellationException import org.koin.core.annotation.Single +import kotlin.time.Clock +import kotlin.time.Duration.Companion.seconds @Single class AttachmentCleanupService( private val journalRepository: UploadJournalRepository, - private val objectStorageRepository: ObjectStorageRepository + private val objectStorageService: ObjectStorageService, + private val attachmentRepository: AttachmentRepository, + private val config: WorkerConfig ) { + suspend fun cleanup(): Int { val ids = suspendedTransaction { @@ -18,9 +25,10 @@ class AttachmentCleanupService( } var deleted = 0 + ids.forEach { id -> try { - objectStorageRepository.delete(id) + objectStorageService.delete(id) deleted++ } catch (cause: CancellationException) { throw cause @@ -29,6 +37,34 @@ class AttachmentCleanupService( } } + val expired = suspendedTransaction { + attachmentRepository.findExpiredStorage( + before = Clock.System.now() - config.stagingRetentionSeconds.seconds + ) + } + + expired.forEach { attachment -> + try { + when { + attachment.failed && !attachment.legacyOriginal -> + objectStorageService.delete(attachment.id) + attachment.failed -> + objectStorageService.deleteThumbnail(attachment.id) + !attachment.legacyOriginal -> + objectStorageService.deleteStaging(attachment.id) + } + + suspendedTransaction { + attachmentRepository.markStagingCleaned(attachment.id) + } + } catch (cause: CancellationException) { + throw cause + } catch (_: Exception) { + // Terminal metadata remains available for a later cleanup retry. + } + } + return deleted } + } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentService.kt index b4fffe56..cdc0e0b9 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentService.kt @@ -24,5 +24,9 @@ interface AttachmentService { actorId: UUID ): Attachment - suspend fun content(attachment: Attachment): ByteArray + suspend fun content( + attachment: Attachment, + variant: String = "original" + ): ByteArray + } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt new file mode 100644 index 00000000..29812a17 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentProcessingService.kt @@ -0,0 +1,171 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.mtusi.messenger.main.config.worker.WorkerConfig +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.infrastructure.dto.AttachmentVariant +import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.JobClaim +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService +import com.mtusi.messenger.main.infrastructure.types.ApiException +import com.mtusi.messenger.main.infrastructure.types.ErrorCode +import com.mtusi.messenger.main.plugins.suspendedTransaction +import com.mtusi.messenger.main.utils.computeSha256 +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.delay +import kotlinx.coroutines.withTimeoutOrNull +import org.koin.core.annotation.Single +import java.util.UUID +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +@Single +class DefaultAttachmentProcessingService( + private val jobRepository: AttachmentJobRepository, + private val attachmentRepository: AttachmentRepository, + private val objectStorageService: ObjectStorageService, + private val imageProcessingService: ImageProcessingService, + private val jobBrokerService: JobBrokerService, + private val config: WorkerConfig +) { + + suspend fun process( + job: AttachmentJob + ): Boolean { + + val token = UUID.randomUUID() + val claim = suspendedTransaction { + jobRepository.claim( + job = job, + token = token, + maxAttempts = config.maxAttempts, + leaseSeconds = config.timeoutSeconds + 15 + ) + } + + when (claim) { + JobClaim.Complete -> return true + + JobClaim.Busy -> { + // Pace redeliveries while the lease is active. Publish to the tail and + // confirm before acknowledging the original delivery. + delay(250.milliseconds) + jobBrokerService.publish(job) + + return true + } + + is JobClaim.Retry -> { + jobBrokerService.publish(job.copy(attempt = claim.attempt)) + + return true + } + + is JobClaim.Acquired -> Unit + } + + try { + val completed = withTimeoutOrNull(config.timeoutSeconds.seconds) { + processAttachment( + job = job, + token = token, + legacyOriginal = claim.legacyOriginal + ) + } ?: throw ProcessingTimeoutException() + + return completed + } catch (cause: CancellationException) { + // Leave the delivery unacknowledged; a new worker can reclaim the expired lease. + throw cause + } catch (cause: Exception) { + val isPermanent = cause is ApiException && cause.code in setOf( + ErrorCode.UNSUPPORTED_MEDIA_TYPE, + ErrorCode.PAYLOAD_TOO_LARGE + ) + + val errorCode = when { + isPermanent -> "invalid_image" + cause is ProcessingTimeoutException -> "processing_timeout" + cause is ApiException && cause.code == ErrorCode.UNAVAILABLE -> "storage_unavailable" + else -> "processing_failed" + } + + val isTerminal = suspendedTransaction { + jobRepository.fail( + job = job, + token = token, + errorCode = errorCode, + permanent = isPermanent, + maxAttempts = config.maxAttempts + ) + } + + if (!isTerminal) { + jobBrokerService.publish(job.copy(attempt = claim.attempt + 1)) + } + + return true + } + } + + private class ProcessingTimeoutException : RuntimeException() + + private suspend fun processAttachment( + job: AttachmentJob, + token: UUID, + legacyOriginal: Boolean + ): Boolean { + + val attachment = suspendedTransaction { + checkNotNull(attachmentRepository.findById(job.attachmentId)) + } + + val bytes = if (legacyOriginal) { + objectStorageService.get(attachment.id) + } else { + objectStorageService.getStaging(attachment.id) + } + + if (bytes.size.toLong() != attachment.sizeBytes || bytes.computeSha256() != attachment.sha256) { + throw ApiException( + code = ErrorCode.UNSUPPORTED_MEDIA_TYPE, + message = "invalid source image" + ) + } + + val thumbnail = imageProcessingService.thumbnail( + bytes = bytes, + declaredType = attachment.contentType + ) + + if (!legacyOriginal) { + objectStorageService.put( + id = attachment.id, + bytes = bytes, + contentType = attachment.contentType + ) + } + + objectStorageService.putThumbnail( + id = attachment.id, + bytes = thumbnail.bytes + ) + + return suspendedTransaction { + jobRepository.complete( + job = job, + token = token, + variant = AttachmentVariant( + name = "thumbnail", + contentType = thumbnail.contentType, + sizeBytes = thumbnail.bytes.size.toLong(), + width = thumbnail.width, + height = thumbnail.height, + sha256 = thumbnail.bytes.computeSha256() + ) + ) + } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt index ea95a32c..8782680c 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultAttachmentService.kt @@ -1,16 +1,21 @@ package com.mtusi.messenger.main.infrastructure.services.attachment import com.mtusi.messenger.main.infrastructure.dto.Attachment -import com.mtusi.messenger.main.infrastructure.repositories.attachment.UploadJournalRepository +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository -import com.mtusi.messenger.main.infrastructure.repositories.attachment.storage.ObjectStorageRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.UploadJournalRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository import com.mtusi.messenger.main.infrastructure.repositories.chat.ChatRepository +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ApiException import com.mtusi.messenger.main.infrastructure.types.ErrorCode import com.mtusi.messenger.main.plugins.suspendedTransaction import com.mtusi.messenger.main.utils.computeSha256 import com.mtusi.messenger.main.utils.nowMicros +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.withContext import org.koin.core.annotation.Single import java.io.File @@ -22,14 +27,24 @@ class DefaultAttachmentService( private val journalRepository: UploadJournalRepository, private val attachmentRepository: AttachmentRepository, private val chatRepository: ChatRepository, - private val objectStorageRepository: ObjectStorageRepository, + private val objectStorageService: ObjectStorageService, + private val jobRepository: AttachmentJobRepository, + private val jobBrokerService: JobBrokerService ) : AttachmentService { override suspend fun requireMember( chatId: UUID, actorId: UUID ) { - if (!suspendedTransaction { chatRepository.isMember(chatId, actorId) }) { + + val isMember = suspendedTransaction { + chatRepository.isMember( + chatId = chatId, + userId = actorId + ) + } + + if (!isMember) { notFound() } } @@ -42,11 +57,16 @@ class DefaultAttachmentService( file: File ): Attachment { - requireMember(chatId, actorId) + requireMember( + chatId = chatId, + actorId = actorId + ) - val (type, hash) = withContext(Dispatchers.IO) { + val (validatedType, sha256) = withContext(Dispatchers.IO) { val bytes = file.readBytes() - ImageValidator.validate(bytes, contentType) to bytes.computeSha256() + val type = ImageValidator.validate(bytes, contentType) + + type to bytes.computeSha256() } val attachment = Attachment( @@ -54,24 +74,62 @@ class DefaultAttachmentService( chatId = chatId, ownerId = actorId, filename = filename, - contentType = type, + contentType = validatedType, sizeBytes = file.length(), - sha256 = hash, - createdAt = Clock.System.nowMicros() + sha256 = sha256, + createdAt = Clock.System.nowMicros(), + status = "queued" ) suspendedTransaction { journalRepository.register(attachment.id) } - objectStorageRepository.put(attachment.id, file, type) - suspendedTransaction { - if (!journalRepository.lockForCompletion(attachment.id)) { - throw ApiException(ErrorCode.UNAVAILABLE, "upload expired") + objectStorageService.putStaging( + id = attachment.id, + file = file, + contentType = validatedType + ) + + val job = AttachmentJob( + jobId = UUID.randomUUID(), + attachmentId = attachment.id, + createdAt = attachment.createdAt + ) + + try { + suspendedTransaction { + if (!journalRepository.lockForCompletion(attachment.id)) { + throw ApiException( + code = ErrorCode.UNAVAILABLE, + message = "upload expired" + ) + } + + attachmentRepository.create(attachment) + jobRepository.create(job) + journalRepository.complete(attachment.id) } - attachmentRepository.create(attachment) - journalRepository.complete(attachment.id) + + jobBrokerService.publish(job) + } catch (cause: CancellationException) { + withContext(NonCancellable) { + runCatching { + suspendedTransaction { + attachmentRepository.markEnqueueFailed(attachment.id) + } + } + } + + throw cause + } catch (cause: Exception) { + suspendedTransaction { + attachmentRepository.markEnqueueFailed(attachment.id) + } + + throw cause } + return attachment } @@ -79,16 +137,49 @@ class DefaultAttachmentService( id: UUID, actorId: UUID ): Attachment = suspendedTransaction { - val attachment = attachmentRepository.findById(id) ?: notFound() - if (!chatRepository.isMember(attachment.chatId, actorId)) { + + val attachment = attachmentRepository.findById(id) + ?: notFound() + + val isMember = chatRepository.isMember( + chatId = attachment.chatId, + userId = actorId + ) + + if (!isMember) { notFound() } + attachment } - override suspend fun content(attachment: Attachment): ByteArray = withContext(Dispatchers.IO) { - objectStorageRepository.get(attachment.id) + override suspend fun content( + attachment: Attachment, + variant: String + ): ByteArray = withContext(Dispatchers.IO) { + + if (attachment.status != "ready") { + throw ApiException( + code = ErrorCode.CONFLICT, + message = "attachment is not ready" + ) + } + + when (variant) { + "original" -> objectStorageService.get(attachment.id) + "thumbnail" -> objectStorageService.getThumbnail(attachment.id) + else -> throw ApiException( + code = ErrorCode.NOT_FOUND, + message = "variant not found" + ) + } + } + + private fun notFound(): Nothing { + throw ApiException( + code = ErrorCode.NOT_FOUND, + message = "attachment or chat not found" + ) } - private fun notFound(): Nothing = throw ApiException(ErrorCode.NOT_FOUND, "attachment or chat not found") } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt new file mode 100644 index 00000000..8dd1a869 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/DefaultImageProcessingService.kt @@ -0,0 +1,206 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.drew.imaging.jpeg.JpegMetadataReader +import com.drew.lang.ByteArrayReader +import com.drew.metadata.Metadata +import com.drew.metadata.exif.ExifIFD0Directory +import com.drew.metadata.exif.ExifReader +import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.withContext +import org.koin.core.annotation.Single +import java.awt.Color +import java.awt.RenderingHints +import java.awt.geom.AffineTransform +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import javax.imageio.ImageIO + +private const val THUMBNAIL_SIZE = 256 + +@Single +class DefaultImageProcessingService : ImageProcessingService { + + override suspend fun thumbnail( + bytes: ByteArray, + declaredType: String + ): ProcessedImage = withContext(Dispatchers.IO) { + + currentCoroutineContext().ensureActive() + + val original = ImageValidator.decode( + bytes = bytes, + declaredType = declaredType + ) + + try { + currentCoroutineContext().ensureActive() + + val orientation = readOrientation(bytes, declaredType) + val swapDimensions = orientation in 5..8 + val orientedWidth = if (swapDimensions) original.height else original.width + val orientedHeight = if (swapDimensions) original.width else original.height + val longestSide = maxOf(orientedWidth, orientedHeight) + + val width = if (longestSide <= THUMBNAIL_SIZE) { + orientedWidth + } else { + maxOf(1, (orientedWidth.toLong() * THUMBNAIL_SIZE / longestSide).toInt()) + } + + val height = if (longestSide <= THUMBNAIL_SIZE) { + orientedHeight + } else { + maxOf(1, (orientedHeight.toLong() * THUMBNAIL_SIZE / longestSide).toInt()) + } + + val thumbnail = BufferedImage(width, height, BufferedImage.TYPE_INT_RGB) + + try { + val transform = AffineTransform.getScaleInstance( + width.toDouble() / orientedWidth, + height.toDouble() / orientedHeight + ).apply { + val orientationCorrection = orientationTransform( + orientation = orientation, + width = original.width, + height = original.height + ) + + concatenate(orientationCorrection) + } + + drawThumbnail( + original = original, + thumbnail = thumbnail, + transform = transform + ) + + currentCoroutineContext().ensureActive() + + val encoded = ByteArrayOutputStream().use { output -> + check(ImageIO.write(thumbnail, "jpeg", output)) { + "JPEG encoder is unavailable" + } + + output.toByteArray() + } + + ProcessedImage( + bytes = encoded, + width = width, + height = height + ) + } finally { + thumbnail.flush() + } + } finally { + original.flush() + } + } + + private fun drawThumbnail( + original: BufferedImage, + thumbnail: BufferedImage, + transform: AffineTransform + ) { + + val graphics = thumbnail.createGraphics() + + try { + graphics.color = Color.WHITE + graphics.fillRect(0, 0, thumbnail.width, thumbnail.height) + + graphics.setRenderingHint( + RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BILINEAR + ) + + graphics.setRenderingHint( + RenderingHints.KEY_RENDERING, + RenderingHints.VALUE_RENDER_QUALITY + ) + + graphics.drawImage(original, transform, null) + } finally { + graphics.dispose() + } + } + + private fun readOrientation( + bytes: ByteArray, + declaredType: String + ): Int { + + // Read only EXIF: generic PNG metadata readers can inflate optional text chunks. + val metadata = if (declaredType == "image/jpeg") { + ByteArrayInputStream(bytes).use { input -> + JpegMetadataReader.readMetadata(input, listOf(ExifReader())) + } + } else { + readPngExif(bytes) + } + + return metadata + .getFirstDirectoryOfType(ExifIFD0Directory::class.java) + ?.getInteger(ExifIFD0Directory.TAG_ORIENTATION) + ?.takeIf { it in 1..8 } + ?: 1 + } + + private fun readPngExif( + bytes: ByteArray + ): Metadata { + + val metadata = Metadata() + var offset = 8 + + while (offset + 12 <= bytes.size) { + val length = ByteBuffer.wrap(bytes, offset, 4).int + val type = String(bytes, offset + 4, 4, Charsets.US_ASCII) + + if (type == "eXIf") { + val dataOffset = offset + 8 + val exif = ByteArrayReader( + bytes.copyOfRange( + fromIndex = dataOffset, + toIndex = dataOffset + length + ) + ) + + ExifReader().extract(exif, metadata) + break + } + + offset += length + 12 + } + + return metadata + } + + private fun orientationTransform( + orientation: Int, + width: Int, + height: Int + ): AffineTransform { + + val sourceWidth = width.toDouble() + val sourceHeight = height.toDouble() + + return when (orientation) { + 2 -> AffineTransform(-1.0, 0.0, 0.0, 1.0, sourceWidth, 0.0) + 3 -> AffineTransform(-1.0, 0.0, 0.0, -1.0, sourceWidth, sourceHeight) + 4 -> AffineTransform(1.0, 0.0, 0.0, -1.0, 0.0, sourceHeight) + 5 -> AffineTransform(0.0, 1.0, 1.0, 0.0, 0.0, 0.0) + 6 -> AffineTransform(0.0, 1.0, -1.0, 0.0, sourceHeight, 0.0) + 7 -> AffineTransform(0.0, -1.0, -1.0, 0.0, sourceHeight, sourceWidth) + 8 -> AffineTransform(0.0, -1.0, 1.0, 0.0, 0.0, sourceWidth) + else -> AffineTransform() + } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt new file mode 100644 index 00000000..581cc459 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingService.kt @@ -0,0 +1,12 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage + +interface ImageProcessingService { + + suspend fun thumbnail( + bytes: ByteArray, + declaredType: String + ): ProcessedImage + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageValidator.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageValidator.kt index c693f2c2..02f92abf 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageValidator.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageValidator.kt @@ -2,9 +2,10 @@ package com.mtusi.messenger.main.infrastructure.services.attachment import com.mtusi.messenger.main.infrastructure.types.ApiException import com.mtusi.messenger.main.infrastructure.types.ErrorCode +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream import java.nio.ByteBuffer import java.util.zip.CRC32 -import java.io.ByteArrayInputStream import javax.imageio.ImageIO import javax.imageio.stream.MemoryCacheImageInputStream @@ -18,6 +19,19 @@ object ImageValidator { declaredType: String ): String { + decode( + bytes = bytes, + declaredType = declaredType + ).flush() + + return declaredType + } + + fun decode( + bytes: ByteArray, + declaredType: String + ): BufferedImage { + if (bytes.size > MAX_ATTACHMENT_BYTES) { throw ApiException(ErrorCode.PAYLOAD_TOO_LARGE, "file exceeds 10 MiB") } @@ -72,13 +86,13 @@ object ImageValidator { reader.addIIOReadWarningListener { _, _ -> warning = true } val image = reader.read(0) ?: invalid() - image.flush() if (warning) { + image.flush() invalid() } - return type + return image } finally { reader.dispose() } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/JobBrokerService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/JobBrokerService.kt new file mode 100644 index 00000000..38187dd9 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/JobBrokerService.kt @@ -0,0 +1,20 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob + +interface JobBrokerService { + + suspend fun publish( + job: AttachmentJob + ) + + /** A true result acknowledges a durable effect; false requeues the delivery. */ + suspend fun consume( + handler: suspend (AttachmentJob) -> Boolean + ) + + suspend fun ready(): Boolean + + fun close() + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt new file mode 100644 index 00000000..2027d4b4 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/LegacyAttachmentDispatchService.kt @@ -0,0 +1,28 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment.jobs + +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository +import com.mtusi.messenger.main.plugins.suspendedTransaction +import org.koin.core.annotation.Single + +@Single +class LegacyAttachmentDispatchService( + private val jobRepository: AttachmentJobRepository, + private val jobBrokerService: JobBrokerService +) { + + suspend fun publishPending() { + + val jobs = suspendedTransaction { + jobRepository.findPendingLegacyJobs(limit = 32) + } + + for (job in jobs) { + jobBrokerService.publish(job) + + suspendedTransaction { + jobRepository.markLegacyJobDispatched(job.jobId) + } + } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/RabbitMqJobBrokerService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/RabbitMqJobBrokerService.kt new file mode 100644 index 00000000..b934a114 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/RabbitMqJobBrokerService.kt @@ -0,0 +1,314 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment.jobs + +import com.mtusi.messenger.main.config.broker.BrokerConfig +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.infrastructure.types.ApiException +import com.mtusi.messenger.main.infrastructure.types.ErrorCode +import com.mtusi.messenger.main.utils.resolveSecret +import com.rabbitmq.client.AMQP +import com.rabbitmq.client.Channel +import com.rabbitmq.client.Connection +import com.rabbitmq.client.ConnectionFactory +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.isActive +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonNamingStrategy +import org.koin.core.annotation.Single +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +private const val MAX_JOB_BYTES = 16 * 1024 + +@Single +class RabbitMqJobBrokerService( + private val config: BrokerConfig +) : JobBrokerService { + + private val logger = LoggerFactory.getLogger(RabbitMqJobBrokerService::class.java) + private val closed = AtomicBoolean(false) + private val publisherMutex = Mutex() + private val consumers = ConcurrentHashMap.newKeySet() + + @Volatile + private var publisher: Connection? = null + + @OptIn(ExperimentalSerializationApi::class) + private val json = Json { + encodeDefaults = true + namingStrategy = JsonNamingStrategy.SnakeCase + } + + private val factory by lazy { + ConnectionFactory().apply { + host = config.host + port = config.port + username = config.username + password = requireNotNull( + resolveSecret( + value = config.password, + file = config.passwordFile + ) + ) + + connectionTimeout = config.connectionTimeoutMs + handshakeTimeout = config.operationTimeoutMs + channelRpcTimeout = config.operationTimeoutMs + requestedHeartbeat = 10 + isAutomaticRecoveryEnabled = true + networkRecoveryInterval = 1000 + } + } + + override suspend fun publish( + job: AttachmentJob + ) { + + require(valid(job)) { "Unsupported attachment job" } + + try { + publisherMutex.withLock { + withContext(Dispatchers.IO) { + check(!closed.get()) + + val connection = publisherConnection() + + connection.createChannel().use { channel -> + declareQueue(channel) + channel.confirmSelect() + + val returned = AtomicBoolean(false) + channel.addReturnListener { _ -> + returned.set(true) + } + + val properties = AMQP.BasicProperties.Builder() + .contentType("application/json") + .contentEncoding("utf-8") + .deliveryMode(2) + .messageId(job.jobId.toString()) + .build() + + val body = json.encodeToString(job).toByteArray(Charsets.UTF_8) + + channel.basicPublish( + "", + config.queue, + true, + properties, + body + ) + channel.waitForConfirmsOrDie(config.operationTimeoutMs.toLong()) + + check(!returned.get()) { "Attachment job was not routed" } + currentCoroutineContext().ensureActive() + } + } + } + } catch (cause: CancellationException) { + throw cause + } catch (_: Exception) { + currentCoroutineContext().ensureActive() + + throw ApiException( + code = ErrorCode.UNAVAILABLE, + message = "job broker unavailable" + ) + } + } + + override suspend fun consume( + handler: suspend (AttachmentJob) -> Boolean + ) { + withContext(Dispatchers.IO) { + while (currentCoroutineContext().isActive && !closed.get()) { + try { + val connection = connect("messenger-jobs-worker") + consumers.add(connection) + + try { + connection.createChannel().use { channel -> + declareQueue(channel) + channel.basicQos(1) + + consumeDeliveries( + connection = connection, + channel = channel, + handler = handler + ) + } + } finally { + consumers.remove(connection) + runCatching { connection.abort() } + } + } catch (cause: CancellationException) { + throw cause + } catch (cause: Exception) { + currentCoroutineContext().ensureActive() + + if (!closed.get()) { + logger.warn( + "Attachment job consumer disconnected ({}); retrying", + cause.javaClass.simpleName + ) + delay(1.seconds) + } + } + } + } + } + + override suspend fun ready(): Boolean { + return try { + publisherMutex.withLock { + withContext(Dispatchers.IO) { + check(!closed.get()) + + val connection = publisherConnection() + connection.createChannel().use(::declareQueue) + + true + } + } + } catch (cause: CancellationException) { + throw cause + } catch (_: Exception) { + currentCoroutineContext().ensureActive() + + false + } + } + + override fun close() { + closed.set(true) + + runCatching { publisher?.abort() } + + consumers.forEach { connection -> + runCatching { connection.abort() } + } + } + + private suspend fun consumeDeliveries( + connection: Connection, + channel: Channel, + handler: suspend (AttachmentJob) -> Boolean + ) { + + // Waiting for the handler before the next basicGet keeps one delivery in flight. + // Channel or connection closure requeues an unacknowledged job. + while ( + currentCoroutineContext().isActive && + !closed.get() && + connection.isOpen && + channel.isOpen + ) { + val delivery = channel.basicGet(config.queue, false) + + if (delivery == null) { + delay(250.milliseconds) + continue + } + + val job = decode(delivery.body) + val deliveryTag = delivery.envelope.deliveryTag + + if (job == null) { + channel.basicReject(deliveryTag, false) + logger.warn("Rejected unsupported attachment job envelope") + + continue + } + + if (handler(job)) { + channel.basicAck(deliveryTag, false) + } else { + channel.basicNack(deliveryTag, false, true) + delay(500.milliseconds) + } + } + } + + private fun connect( + name: String + ): Connection { + check(!closed.get()) + + return factory.newConnection(name).also { connection -> + if (closed.get()) { + connection.abort() + error("Attachment job broker is closed") + } + } + } + + /** Called under publisherMutex so reconnect does not retain an unused recovery loop. */ + private fun publisherConnection(): Connection { + + publisher?.let { connection -> + if (connection.isOpen) { + return connection + } + + runCatching { connection.abort() } + publisher = null + } + + val connection = connect("messenger-jobs-publisher") + publisher = connection + + if (closed.get()) { + connection.abort() + error("Attachment job broker is closed") + } + + return connection + } + + private fun declareQueue( + channel: Channel + ) { + channel.queueDeclare( + config.queue, + true, + false, + false, + mapOf("x-queue-type" to "classic") + ) + } + + private fun decode( + bytes: ByteArray + ): AttachmentJob? { + + if (bytes.size > MAX_JOB_BYTES) { + return null + } + + return runCatching { + json.decodeFromString(bytes.toString(Charsets.UTF_8)) + } + .getOrNull() + ?.takeIf(::valid) + } + + private fun valid( + job: AttachmentJob + ): Boolean { + return job.schemaVersion == 1 && + job.type == "attachment.process.v1" && + job.pipelineVersion == 1 && + job.attempt >= 1 + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt new file mode 100644 index 00000000..2246ffc6 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/ObjectStorageService.kt @@ -0,0 +1,46 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment.storage + +import java.io.File +import java.util.UUID + +interface ObjectStorageService { + + suspend fun put( + id: UUID, + bytes: ByteArray, + contentType: String + ) + + suspend fun putStaging( + id: UUID, + file: File, + contentType: String + ) + + suspend fun getStaging( + id: UUID + ): ByteArray + + suspend fun deleteStaging( + id: UUID + ) + + suspend fun putThumbnail( + id: UUID, + bytes: ByteArray + ) + + suspend fun getThumbnail( + id: UUID + ): ByteArray + + suspend fun deleteThumbnail( + id: UUID + ) + + suspend fun get(id: UUID): ByteArray + + suspend fun delete(id: UUID) + + suspend fun ready(): Boolean +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/storage/S3ObjectStorageRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt similarity index 61% rename from src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/storage/S3ObjectStorageRepository.kt rename to src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt index 72afac5f..63e3a0fd 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/attachment/storage/S3ObjectStorageRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/storage/S3ObjectStorageService.kt @@ -1,4 +1,4 @@ -package com.mtusi.messenger.main.infrastructure.repositories.attachment.storage +package com.mtusi.messenger.main.infrastructure.services.attachment.storage import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.services.s3.S3Client @@ -22,9 +22,9 @@ import java.util.UUID import kotlin.time.Duration.Companion.milliseconds @Single -class S3ObjectStorageRepository( +class S3ObjectStorageService( private val config: StorageConfig -) : ObjectStorageRepository { +) : ObjectStorageService { private val client by lazy { S3Client { @@ -41,13 +41,33 @@ class S3ObjectStorageRepository( override suspend fun put( id: UUID, - file: File, + bytes: ByteArray, contentType: String ) { request { client.putObject { bucket = config.bucket key = getObjectKey(id) + body = bytes.asByteStream() + this.contentType = contentType + contentLength = bytes.size.toLong() + } + } + } + + override suspend fun get( + id: UUID + ): ByteArray = getBytes(getObjectKey(id)) + + override suspend fun putStaging( + id: UUID, + file: File, + contentType: String + ) { + request { + client.putObject { + bucket = config.bucket + key = getStagingKey(id) body = file.asByteStream() this.contentType = contentType contentLength = file.length() @@ -55,13 +75,72 @@ class S3ObjectStorageRepository( } } - override suspend fun get( + override suspend fun getStaging( id: UUID + ): ByteArray = getBytes(getStagingKey(id)) + + override suspend fun deleteStaging( + id: UUID + ) { + deleteKey(getStagingKey(id)) + } + + override suspend fun putThumbnail( + id: UUID, + bytes: ByteArray + ) { + request { + client.putObject { + bucket = config.bucket + key = getThumbnailKey(id) + body = bytes.asByteStream() + contentType = "image/jpeg" + contentLength = bytes.size.toLong() + } + } + } + + override suspend fun getThumbnail( + id: UUID + ): ByteArray = getBytes(getThumbnailKey(id)) + + override suspend fun deleteThumbnail( + id: UUID + ) { + deleteKey(getThumbnailKey(id)) + } + + override suspend fun delete( + id: UUID + ) { + deleteKey(getObjectKey(id)) + deleteStaging(id) + deleteThumbnail(id) + } + + override suspend fun ready(): Boolean { + return try { + request { + client.headBucket { + bucket = config.bucket + } + } + + true + } catch (_: ApiException) { + false + } + } + + private suspend fun getBytes( + key: String ): ByteArray = request { - client.getObject(GetObjectRequest { - bucket = config.bucket - key = getObjectKey(id) - }) { response -> + client.getObject( + GetObjectRequest { + bucket = config.bucket + this.key = key + } + ) { response -> val contentLength = requireNotNull(response.contentLength) @@ -75,31 +154,28 @@ class S3ObjectStorageRepository( } } - override suspend fun delete( - id: UUID + private suspend fun deleteKey( + key: String ) { request { client.deleteObject { bucket = config.bucket - key = getObjectKey(id) + this.key = key } } } - override suspend fun ready(): Boolean { - return try { - request { - client.headBucket { - bucket = config.bucket - } - } - true - } catch (_: ApiException) { - false - } - } + private fun getObjectKey( + id: UUID + ) = "originals/$id" - private fun getObjectKey(id: UUID) = "originals/$id" + private fun getStagingKey( + id: UUID + ) = "staging/$id" + + private fun getThumbnailKey( + id: UUID + ) = "variants/$id/v1/thumbnail.jpg" private suspend fun request(block: suspend () -> T): T { return try { diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/message/DefaultMessageService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/message/DefaultMessageService.kt index c91ca0ae..1bfd581e 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/message/DefaultMessageService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/message/DefaultMessageService.kt @@ -1,14 +1,14 @@ package com.mtusi.messenger.main.infrastructure.services.message -import com.mtusi.messenger.main.infrastructure.types.ErrorCode -import com.mtusi.messenger.main.infrastructure.types.ApiException -import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository import com.mtusi.messenger.main.config.message.MessageConfig +import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.chat.ChatRepository import com.mtusi.messenger.main.infrastructure.repositories.message.MessageRepository import com.mtusi.messenger.main.infrastructure.repositories.message.idempotency.MessageIdempotencyRepository import com.mtusi.messenger.main.infrastructure.services.message.results.MessageCreationResult import com.mtusi.messenger.main.infrastructure.services.message.results.MessagePageResult +import com.mtusi.messenger.main.infrastructure.types.ApiException +import com.mtusi.messenger.main.infrastructure.types.ErrorCode import com.mtusi.messenger.main.utils.PageKey import com.mtusi.messenger.main.utils.computeSha256 import com.mtusi.messenger.main.utils.decodeCursor @@ -44,14 +44,20 @@ class DefaultMessageService( return MessageCreationResult.ChatNotFound } - attachmentIds.forEach { id -> - if (attachments.findById(id)?.chatId != chatId) - throw ApiException( - ErrorCode.NOT_FOUND, "attachment not found") + requireAttachmentStatuses( + chatId = chatId, + attachmentIds = attachmentIds + ) + + val attachmentSuffix = if (attachmentIds.isEmpty()) { + "" + } else { + "\u0000attachments:" + attachmentIds.joinToString(",") } - val requestHash = (text + if (attachmentIds.isEmpty()) "" else "\u0000attachments:" + attachmentIds.joinToString(",")) - .toByteArray().computeSha256() + val requestHash = (text + attachmentSuffix) + .toByteArray() + .computeSha256() val now = Clock.System.nowMicros() @@ -59,6 +65,19 @@ class DefaultMessageService( if (idempotencyRepository.claim(senderId, chatId, idempotencyKey, requestHash, now)) { + val statuses = requireAttachmentStatuses( + chatId = chatId, + attachmentIds = attachmentIds, + lockForUpdate = true + ) + + if ("failed" in statuses) { + throw ApiException( + code = ErrorCode.CONFLICT, + message = "attachment processing failed" + ) + } + val message = messageRepository.create( chatId = chatId, senderId = senderId, @@ -103,6 +122,28 @@ class DefaultMessageService( error("Idempotency key could not be claimed") } + private fun requireAttachmentStatuses( + chatId: UUID, + attachmentIds: List, + lockForUpdate: Boolean = false + ): List { + + val statuses = attachments.findStatusesInChat( + chatId = chatId, + ids = attachmentIds, + lockForUpdate = lockForUpdate + ) + + if (statuses.size != attachmentIds.size) { + throw ApiException( + code = ErrorCode.NOT_FOUND, + message = "attachment not found" + ) + } + + return statuses + } + override fun findPageByChatId( chatId: UUID, actorId: UUID, diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentCleanup.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentCleanup.kt index 76f69853..1c39666d 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentCleanup.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentCleanup.kt @@ -2,6 +2,7 @@ package com.mtusi.messenger.main.plugins import com.mtusi.messenger.main.infrastructure.services.attachment.AttachmentCleanupService import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationStopping import io.ktor.server.application.log import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay @@ -14,9 +15,10 @@ fun Application.configureAttachmentCleanup() { val cleanupService by inject() - launch { + val cleanup = launch { while (isActive) { delay(30.seconds) + try { cleanupService.cleanup() } catch (cause: CancellationException) { @@ -26,4 +28,9 @@ fun Application.configureAttachmentCleanup() { } } } + + monitor.subscribe(ApplicationStopping) { + cleanup.cancel() + } + } diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt new file mode 100644 index 00000000..d2c94b20 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/AttachmentWorker.kt @@ -0,0 +1,50 @@ +package com.mtusi.messenger.main.plugins + +import com.mtusi.messenger.main.config.worker.WorkerConfig +import com.mtusi.messenger.main.infrastructure.services.attachment.DefaultAttachmentProcessingService +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.LegacyAttachmentDispatchService +import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationStopping +import io.ktor.server.application.log +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import org.koin.ktor.ext.get +import kotlin.time.Duration.Companion.seconds + +fun Application.configureAttachmentWorker() { + + val broker = get() + val processingService = get() + val legacyDispatchService = get() + val config = get() + + val workers = List(config.concurrency) { + launch { + broker.consume(processingService::process) + } + } + + val legacyDispatch = launch { + while (isActive) { + try { + legacyDispatchService.publishPending() + } catch (cause: CancellationException) { + throw cause + } catch (cause: Exception) { + log.warn("Legacy attachment dispatch failed ({}); retrying", cause.javaClass.simpleName) + } + + delay(1.seconds) + } + } + + monitor.subscribe(ApplicationStopping) { + workers.forEach { it.cancel() } + legacyDispatch.cancel() + broker.close() + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt index d54e0b83..9a55309e 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt @@ -1,10 +1,12 @@ package com.mtusi.messenger.main.plugins -import com.mtusi.messenger.main.infrastructure.models.attachment.MessageAttachmentModel -import com.mtusi.messenger.main.infrastructure.models.attachment.UploadJournalModel -import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel import com.mtusi.messenger.main.config.database.DatabaseConfig import com.mtusi.messenger.main.config.database.SerialDatabaseConfig +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentVariantModel +import com.mtusi.messenger.main.infrastructure.models.attachment.MessageAttachmentModel +import com.mtusi.messenger.main.infrastructure.models.attachment.UploadJournalModel import com.mtusi.messenger.main.infrastructure.models.authentication.SessionModel import com.mtusi.messenger.main.infrastructure.models.authentication.SessionRevocationModel import com.mtusi.messenger.main.infrastructure.models.chat.ChatMemberModel @@ -21,18 +23,19 @@ import io.ktor.server.application.ApplicationStopped import io.ktor.server.config.HoconApplicationConfig import io.ktor.server.config.getAs import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.flywaydb.core.Flyway import org.jetbrains.exposed.v1.core.ExperimentalDatabaseMigrationApi import org.jetbrains.exposed.v1.core.Table -import org.jetbrains.exposed.v1.core.DatabaseConfig as ExposedDatabaseConfig import org.jetbrains.exposed.v1.jdbc.Database import org.jetbrains.exposed.v1.jdbc.JdbcTransaction -import org.jetbrains.exposed.v1.jdbc.transactions.experimental.newSuspendedTransaction +import org.jetbrains.exposed.v1.jdbc.transactions.suspendTransaction import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.migration.jdbc.MigrationUtils import org.koin.ktor.ext.inject import java.io.File import javax.sql.DataSource +import org.jetbrains.exposed.v1.core.DatabaseConfig as ExposedDatabaseConfig private const val MIGRATIONS_DIRECTORY = "src/main/resources/migrations" private const val DEVELOPMENT_DATABASE_CONFIG = "run/development/config/database.conf" @@ -121,6 +124,8 @@ private fun generateMigration(description: String) { val tables: Array = arrayOf( AttachmentModel, + AttachmentVariantModel, + AttachmentJobModel, UploadJournalModel, MessageAttachmentModel, UserModel, @@ -175,7 +180,10 @@ private fun generateMigration(description: String) { } -suspend fun suspendedTransaction(statement: JdbcTransaction.() -> R) = newSuspendedTransaction( - context = Dispatchers.IO, - statement = statement -) +suspend fun suspendedTransaction( + statement: JdbcTransaction.() -> R +): R = withContext(Dispatchers.IO) { + suspendTransaction { + statement() + } +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/JobBroker.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/JobBroker.kt new file mode 100644 index 00000000..536caadb --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/JobBroker.kt @@ -0,0 +1,16 @@ +package com.mtusi.messenger.main.plugins + +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationStopping +import org.koin.ktor.ext.get + +fun Application.configureJobBroker() { + + val broker = get() + + monitor.subscribe(ApplicationStopping) { + broker.close() + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/SessionRevocationSync.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/SessionRevocationSync.kt index b0c7343e..6f389c0d 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/plugins/SessionRevocationSync.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/SessionRevocationSync.kt @@ -2,6 +2,7 @@ package com.mtusi.messenger.main.plugins import com.mtusi.messenger.main.infrastructure.services.authentication.AuthenticationService import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationStopping import io.ktor.server.application.log import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay @@ -21,9 +22,10 @@ fun Application.configureSessionRevocationSync() { val authenticationService by inject() - launch { + val synchronization = launch { while (isActive) { delay(SYNC_INTERVAL) + try { authenticationService.syncPendingRevocations() } catch (e: CancellationException) { @@ -33,4 +35,9 @@ fun Application.configureSessionRevocationSync() { } } } + + monitor.subscribe(ApplicationStopping) { + synchronization.cancel() + } + } diff --git a/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt b/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt new file mode 100644 index 00000000..a326856a --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/worker/Worker.kt @@ -0,0 +1,38 @@ +package com.mtusi.messenger.main.worker + +import com.mtusi.messenger.main.infrastructure.controllers.health.configureHealthRouting +import com.mtusi.messenger.main.plugins.configureAttachmentCleanup +import com.mtusi.messenger.main.plugins.configureAttachmentWorker +import com.mtusi.messenger.main.plugins.configureContentNegotiation +import com.mtusi.messenger.main.plugins.configureDatabase +import com.mtusi.messenger.main.plugins.configureDefaultHeaders +import com.mtusi.messenger.main.plugins.configureGracefulShutdown +import com.mtusi.messenger.main.plugins.configureInstanceId +import com.mtusi.messenger.main.plugins.configureJobBroker +import com.mtusi.messenger.main.plugins.configureKoin +import com.mtusi.messenger.main.plugins.configureRequestId +import com.mtusi.messenger.main.plugins.configureStatusPages +import io.ktor.server.application.Application +import io.ktor.server.routing.routing + +fun Application.workerModule() { + + configureKoin() + configureDatabase() + configureGracefulShutdown() + + configureContentNegotiation() + configureDefaultHeaders() + configureInstanceId() + configureRequestId() + configureStatusPages() + + configureJobBroker() + configureAttachmentWorker() + configureAttachmentCleanup() + + routing { + configureHealthRouting() + } + +} diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 4d51dca4..c504aa1a 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -110,3 +110,29 @@ storage { accessKeyFile = ${?S3_ACCESS_KEY_ID_FILE} secretKeyFile = ${?S3_SECRET_ACCESS_KEY_FILE} } + +broker { + host = "localhost" + host = ${?BROKER_HOST} + port = 5672 + port = ${?BROKER_PORT} + username = "messenger" + username = ${?BROKER_USER} + password = ${?BROKER_PASSWORD} + passwordFile = ${?BROKER_PASSWORD_FILE} + queue = "attachment.process.v1" + queue = ${?BROKER_QUEUE} + connectionTimeoutMs = 3000 + operationTimeoutMs = 8000 +} + +worker { + concurrency = 1 + concurrency = ${?WORKER_CONCURRENCY} + maxAttempts = 5 + maxAttempts = ${?JOB_MAX_ATTEMPTS} + timeoutSeconds = 60 + timeoutSeconds = ${?JOB_TIMEOUT_SECONDS} + stagingRetentionSeconds = 86400 + stagingRetentionSeconds = ${?STAGING_RETENTION_SECONDS} +} diff --git a/src/main/resources/migrations/V6__attachment_processing.sql b/src/main/resources/migrations/V6__attachment_processing.sql new file mode 100644 index 00000000..e2808304 --- /dev/null +++ b/src/main/resources/migrations/V6__attachment_processing.sql @@ -0,0 +1,29 @@ +ALTER TABLE attachments ADD COLUMN status VARCHAR(16) NOT NULL DEFAULT 'ready' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD COLUMN staging_cleaned BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE attachments ADD COLUMN error_code VARCHAR(64); +ALTER TABLE attachments ADD COLUMN updated_at TIMESTAMPTZ; +UPDATE attachments SET updated_at = created_at; +ALTER TABLE attachments ALTER COLUMN updated_at SET NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + name VARCHAR(16) NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + width INTEGER NOT NULL CHECK (width > 0), + height INTEGER NOT NULL CHECK (height > 0), + sha256 VARCHAR(64) NOT NULL, + PRIMARY KEY (attachment_id, name) +); + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL DEFAULT 1, + attempts INTEGER NOT NULL DEFAULT 0, + lease_token UUID, + lease_until TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + CHECK ((lease_token IS NULL) = (lease_until IS NULL)) +); diff --git a/src/main/resources/migrations/V7__attachment_retry_metadata.sql b/src/main/resources/migrations/V7__attachment_retry_metadata.sql new file mode 100644 index 00000000..b6c6c92d --- /dev/null +++ b/src/main/resources/migrations/V7__attachment_retry_metadata.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachment_jobs + ADD COLUMN last_error_code VARCHAR(64), + ADD COLUMN next_attempt_at TIMESTAMPTZ; diff --git a/src/main/resources/migrations/V8__backfill_attachment_thumbnails.sql b/src/main/resources/migrations/V8__backfill_attachment_thumbnails.sql new file mode 100644 index 00000000..f3997be3 --- /dev/null +++ b/src/main/resources/migrations/V8__backfill_attachment_thumbnails.sql @@ -0,0 +1,19 @@ +ALTER TABLE attachment_jobs ADD COLUMN legacy_original BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE attachment_jobs ADD COLUMN legacy_dispatch_pending BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE INDEX attachment_jobs_legacy_dispatch_idx ON attachment_jobs (created_at, id) + WHERE legacy_dispatch_pending; + +INSERT INTO attachment_jobs (id, attachment_id, created_at, legacy_original, legacy_dispatch_pending) +SELECT a.id, a.id, CURRENT_TIMESTAMP, TRUE, TRUE +FROM attachments a +WHERE a.status = 'ready' + AND NOT EXISTS (SELECT 1 FROM attachment_jobs j WHERE j.attachment_id = a.id) + AND NOT EXISTS ( + SELECT 1 FROM attachment_variants v WHERE v.attachment_id = a.id AND v.name = 'thumbnail' + ); + +UPDATE attachments a +SET status = 'queued', error_code = NULL, updated_at = CURRENT_TIMESTAMP, staging_cleaned = FALSE +FROM attachment_jobs j +WHERE j.attachment_id = a.id AND j.legacy_dispatch_pending; diff --git a/src/main/resources/worker.conf b/src/main/resources/worker.conf new file mode 100644 index 00000000..b3da2833 --- /dev/null +++ b/src/main/resources/worker.conf @@ -0,0 +1,2 @@ +include "application.conf" +ktor.application.modules = [com.mtusi.messenger.main.worker.WorkerKt.workerModule] diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt index c8181d20..400a8e2e 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/controllers/attachment/AttachmentControllerTest.kt @@ -1,27 +1,53 @@ package com.mtusi.messenger.main.infrastructure.controllers.attachment +import com.mtusi.messenger.main.infrastructure.controllers.message.requests.CreateMessageRequest import com.mtusi.messenger.main.infrastructure.dto.Attachment import com.mtusi.messenger.main.infrastructure.dto.Message -import com.mtusi.messenger.main.infrastructure.controllers.message.requests.CreateMessageRequest -import com.mtusi.messenger.main.support.* +import com.mtusi.messenger.main.support.TestStorage +import com.mtusi.messenger.main.support.TestUser +import com.mtusi.messenger.main.support.auth +import com.mtusi.messenger.main.support.createChat +import com.mtusi.messenger.main.support.idempotencyKey +import com.mtusi.messenger.main.support.register +import com.mtusi.messenger.main.support.runApp +import io.ktor.client.HttpClient import io.ktor.client.call.body -import io.ktor.client.request.* -import io.ktor.http.* +import io.ktor.client.request.get +import io.ktor.client.request.parameter +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.http.ContentType +import io.ktor.http.Headers +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType import io.ktor.http.content.OutgoingContent +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout import java.awt.image.BufferedImage import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.util.UUID +import java.util.zip.CRC32 import javax.imageio.ImageIO -import kotlin.test.* +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds class AttachmentControllerTest { @Test - fun `upload originals ACL message links and idempotency`() = runApp { client -> + val alice = client.register("alice") val bob = client.register("bob") val outsider = client.register("outsider") val chat = client.createChat(alice, "files", listOf(alice.id, bob.id)) + for (format in listOf("png", "jpeg")) { val bytes = image(format) val upload = client.post("/api/v1/chats/${chat.id}/attachments?filename=photo.$format") { @@ -29,52 +55,101 @@ class AttachmentControllerTest { contentType(ContentType.parse("image/$format")) setBody(bytes) } - assertEquals(HttpStatusCode.Created, upload.status) + + assertEquals(HttpStatusCode.Accepted, upload.status) + val attachment = upload.body() - assertEquals("ready", attachment.status) + + assertEquals("queued", attachment.status) + assertEquals(emptyList(), attachment.variants) assertEquals(bytes.size.toLong(), attachment.sizeBytes) + + val ready = client.awaitReady(attachment.id, bob) + + assertEquals(listOf("thumbnail"), ready.variants.map { it.name }) + val path = "/api/v1/attachments/${attachment.id}" val download = client.get("$path/content") { auth(bob) } + assertEquals(HttpStatusCode.OK, download.status) assertContentEquals(bytes, download.body()) assertEquals(HttpStatusCode.NotFound, client.get(path) { auth(outsider) }.status) assertEquals(HttpStatusCode.NotFound, client.get("$path/content") { auth(outsider) }.status) assertEquals(HttpStatusCode.Unauthorized, client.get("$path/content").status) - assertEquals(HttpStatusCode.NotFound, client.get("$path/content?variant=thumbnail") { auth(bob) }.status) + + val thumbnail = client.get("$path/content?variant=thumbnail") { auth(bob) } + + assertEquals(HttpStatusCode.OK, thumbnail.status) + assertEquals(ContentType.Image.JPEG, thumbnail.contentType()?.withoutParameters()) + + val thumbnailBytes = thumbnail.body() + val decoded = withContext(Dispatchers.IO) { + thumbnailBytes.inputStream().use { input -> + ImageIO.read(input) + } + } + val missingVariant = client.get("$path/content?variant=crop") { auth(bob) } + + assertEquals(32, decoded.width) + assertEquals(20, decoded.height) + assertEquals(HttpStatusCode.NotFound, missingVariant.status) + val outsiderChat = client.createChat(outsider, "outsider", listOf(outsider.id)) - assertEquals(HttpStatusCode.NotFound, client.post("/api/v1/chats/${outsiderChat.id}/messages") { + val rejectedByMembership = client.post("/api/v1/chats/${outsiderChat.id}/messages") { auth(outsider) idempotencyKey() contentType(ContentType.Application.Json) - setBody(CreateMessageRequest("", listOf(attachment.id.toString()))) - }.status) + setBody( + CreateMessageRequest( + text = "", + attachmentIds = listOf(attachment.id.toString()) + ) + ) + } + + assertEquals(HttpStatusCode.NotFound, rejectedByMembership.status) + val key = "file-$format" + suspend fun send(ids: List) = client.post("/api/v1/chats/${chat.id}/messages") { auth(bob) idempotencyKey(key) contentType(ContentType.Application.Json) - setBody(CreateMessageRequest("", ids)) + setBody(CreateMessageRequest(text = "", attachmentIds = ids)) } + val first = send(listOf(attachment.id.toString())) + assertEquals(HttpStatusCode.Created, first.status) + val message = first.body() + assertEquals(listOf(attachment.id), message.attachmentIds) assertEquals(message, send(listOf(attachment.id.toString())).body()) + val otherChat = client.createChat(bob, "other", listOf(bob.id)) - assertEquals(HttpStatusCode.NotFound, client.post("/api/v1/chats/${otherChat.id}/messages") { + val rejectedByChat = client.post("/api/v1/chats/${otherChat.id}/messages") { auth(bob) idempotencyKey() contentType(ContentType.Application.Json) - setBody(CreateMessageRequest("", listOf(attachment.id.toString()))) - }.status) + setBody( + CreateMessageRequest( + text = "", + attachmentIds = listOf(attachment.id.toString()) + ) + ) + } + + assertEquals(HttpStatusCode.NotFound, rejectedByChat.status) } } @Test - fun `invalid images filenames and byte limit return client errors`() = runApp { client -> + val user = client.register("upload") val chat = client.createChat(user, "files", listOf(user.id)) + suspend fun upload( bytes: ByteArray, type: String = "image/png", @@ -86,6 +161,7 @@ class AttachmentControllerTest { contentType(ContentType.parse(type)) setBody(bytes) } + val malformedMime = client.post("/api/v1/chats/${chat.id}/attachments?filename=photo.png") { auth(user) // Keep the invalid header in raw content so the client does not parse it first. @@ -96,30 +172,73 @@ class AttachmentControllerTest { override fun bytes() = image("png") }) } + assertEquals(HttpStatusCode.BadRequest, malformedMime.status) assertEquals(HttpStatusCode.UnsupportedMediaType, upload("not an image".toByteArray()).status) assertEquals(HttpStatusCode.UnsupportedMediaType, upload(image("png"), "image/jpeg").status) assertEquals(HttpStatusCode.UnsupportedMediaType, upload(image("png").copyOf(40)).status) - for (name in listOf("", " ", "\u2003", "../x.png", "a\\b.png", "a\n.png", "a\u0000.png", "x".repeat(256))) + + val corrupt = image("png") + var offset = 8 + + while (offset + 12 <= corrupt.size) { + val length = ByteBuffer.wrap(corrupt, offset, 4).int + val type = String(corrupt, offset + 4, 4, Charsets.US_ASCII) + + if (type == "IDAT") { + corrupt[offset + 8] = 0 // Invalid zlib header, but a correct PNG chunk CRC. + val crc = CRC32().apply { + update(corrupt, offset + 4, length + 4) + } + + ByteBuffer.wrap(corrupt, offset + 8 + length, 4) + .putInt(crc.value.toInt()) + break + } + + offset += length + 12 + } + + assertEquals(HttpStatusCode.UnsupportedMediaType, upload(corrupt).status) + + val invalidNames = listOf( + "", + " ", + "\u2003", + "../x.png", + "a\\b.png", + "a\n.png", + "a\u0000.png", + "x".repeat(256) + ) + + for (name in invalidNames) { assertEquals(HttpStatusCode.BadRequest, upload(image("png"), name = name).status) + } + assertEquals(HttpStatusCode.PayloadTooLarge, upload(ByteArray(10 * 1024 * 1024 + 1)).status) } @Test - fun `S3 outage fails upload download and readiness without losing the original`() = runApp { client -> + val user = client.register("outage") val chat = client.createChat(user, "files", listOf(user.id)) val bytes = image("png") + suspend fun upload() = client.post("/api/v1/chats/${chat.id}/attachments?filename=photo.png") { auth(user) contentType(ContentType.Image.PNG) setBody(bytes) } + val attachment = upload().body() + client.awaitReady(attachment.id, user) + val path = "/api/v1/attachments/${attachment.id}/content" val container = TestStorage.container container.dockerClient.pauseContainerCmd(container.containerId).exec() + try { assertEquals(HttpStatusCode.ServiceUnavailable, upload().status) assertEquals(HttpStatusCode.ServiceUnavailable, client.get(path) { auth(user) }.status) @@ -128,10 +247,27 @@ class AttachmentControllerTest { } finally { container.dockerClient.unpauseContainerCmd(container.containerId).exec() } + assertContentEquals(bytes, client.get(path) { auth(user) }.body()) assertEquals(HttpStatusCode.OK, client.get("/health/ready").status) } + private suspend fun HttpClient.awaitReady( + id: UUID, + user: TestUser + ): Attachment = withTimeout(30.seconds) { + val path = "/api/v1/attachments/$id" + var attachment = get(path) { auth(user) }.body() + + while (attachment.status != "ready") { + check(attachment.status != "failed") { "attachment failed: ${attachment.errorCode}" } + delay(50.milliseconds) + attachment = get(path) { auth(user) }.body() + } + + attachment + } + private fun image(format: String): ByteArray = ByteArrayOutputStream().use { ImageIO.write(BufferedImage(32, 20, BufferedImage.TYPE_INT_RGB), format, it) it.toByteArray() diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentJobContractTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentJobContractTest.kt new file mode 100644 index 00000000..5fcb07ca --- /dev/null +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/dto/AttachmentJobContractTest.kt @@ -0,0 +1,54 @@ +package com.mtusi.messenger.main.infrastructure.dto + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonNamingStrategy +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import java.util.UUID +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.time.Clock + +class AttachmentJobContractTest { + + @OptIn(ExperimentalSerializationApi::class) + @Test + fun `job envelope requires every field mandated by the public schema`() { + + val json = Json { + encodeDefaults = true + namingStrategy = JsonNamingStrategy.SnakeCase + } + val job = AttachmentJob( + jobId = UUID.randomUUID(), + attachmentId = UUID.randomUUID(), + createdAt = Clock.System.now() + ) + + val encoded = json.encodeToString(job) + + assertEquals(job, json.decodeFromString(encoded)) + + val fields = json.parseToJsonElement(encoded).jsonObject + val requiredFields = listOf( + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at" + ) + + for (key in requiredFields) { + val incomplete = JsonObject(fields.filterKeys { it != key }).toString() + + assertFailsWith("Missing required field: $key") { + json.decodeFromString(incomplete) + } + } + } +} diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt index 9f1a4eae..32d6ca01 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentCleanupTest.kt @@ -1,25 +1,25 @@ package com.mtusi.messenger.main.infrastructure.services.attachment import com.mtusi.messenger.main.config.storage.SerialStorageConfig +import com.mtusi.messenger.main.config.worker.SerialWorkerConfig +import com.mtusi.messenger.main.infrastructure.dto.Attachment +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentModel +import com.mtusi.messenger.main.infrastructure.models.attachment.UploadJournalModel import com.mtusi.messenger.main.infrastructure.repositories.attachment.ExposedAttachmentRepository import com.mtusi.messenger.main.infrastructure.repositories.attachment.ExposedUploadJournalRepository -import com.mtusi.messenger.main.infrastructure.repositories.attachment.storage.ObjectStorageRepository -import com.mtusi.messenger.main.infrastructure.repositories.attachment.storage.S3ObjectStorageRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.ExposedAttachmentJobRepository import com.mtusi.messenger.main.infrastructure.repositories.chat.ExposedChatRepository +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.S3ObjectStorageService import com.mtusi.messenger.main.infrastructure.types.ApiException import com.mtusi.messenger.main.plugins.suspendedTransaction import com.mtusi.messenger.main.support.TestStorage import com.mtusi.messenger.main.support.createChat import com.mtusi.messenger.main.support.register import com.mtusi.messenger.main.support.runApp -import com.mtusi.messenger.main.infrastructure.models.attachment.UploadJournalModel import com.mtusi.messenger.main.utils.exposed.operations.plus -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import org.jetbrains.exposed.v1.core.eq -import org.jetbrains.exposed.v1.core.longLiteral -import org.jetbrains.exposed.v1.datetime.CurrentTimestamp -import org.jetbrains.exposed.v1.jdbc.update +import com.mtusi.messenger.main.utils.computeSha256 import java.awt.image.BufferedImage import java.io.File import java.nio.file.Files @@ -31,23 +31,103 @@ import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertTrue +import kotlin.time.Clock +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.longLiteral +import org.jetbrains.exposed.v1.datetime.CurrentTimestamp +import org.jetbrains.exposed.v1.jdbc.update +import org.koin.ktor.ext.get class AttachmentCleanupTest { + @Test + fun `expired failed attachments lose partially published files`() = runApp(startWorker = false) { client -> + + val user = client.register("failed_cleanup") + val chat = client.createChat( + creator = user, + title = "failed cleanup", + memberIds = listOf(user.id) + ) + + val storage = application.get() + val cleanup = application.get() + val bytes = byteArrayOf(1, 2, 3) + val attachment = Attachment( + id = UUID.randomUUID(), + chatId = chat.id, + ownerId = user.id, + filename = "partial.png", + contentType = "image/png", + sizeBytes = bytes.size.toLong(), + sha256 = bytes.computeSha256(), + createdAt = Clock.System.now(), + status = "failed", + errorCode = "retry_exhausted" + ) + + suspendedTransaction { + ExposedAttachmentRepository().create(attachment) + } + + storage.put( + id = attachment.id, + bytes = bytes, + contentType = attachment.contentType + ) + storage.putThumbnail(id = attachment.id, bytes = bytes) + + cleanup.cleanup() + + assertContentEquals(bytes, storage.get(attachment.id)) + assertContentEquals(bytes, storage.getThumbnail(attachment.id)) + + suspendedTransaction { + AttachmentModel.update(where = { AttachmentModel.id eq attachment.id }) { + it[updatedAt] = CurrentTimestamp + longLiteral(-90_000) + } + } + + cleanup.cleanup() + + assertTrue(storage.ready()) + assertFailsWith { storage.get(attachment.id) } + assertFailsWith { storage.getThumbnail(attachment.id) } + } + @Test fun `orphan cleanup respects grace and fences late publishers`() = runApp { client -> val user = client.register("cleanup") val chat = client.createChat(user, "cleanup", listOf(user.id)) + val journal = ExposedUploadJournalRepository() val attachments = ExposedAttachmentRepository() - val storage = S3ObjectStorageRepository(SerialStorageConfig( - endpoint = TestStorage.endpoint, - bucket = TestStorage.BUCKET, - accessKey = TestStorage.ACCESS_KEY, - secretKey = TestStorage.SECRET_KEY - )) - val cleanup = AttachmentCleanupService(journal, storage) + val storage = S3ObjectStorageService( + SerialStorageConfig( + endpoint = TestStorage.endpoint, + bucket = TestStorage.BUCKET, + accessKey = TestStorage.ACCESS_KEY, + secretKey = TestStorage.SECRET_KEY + ) + ) + + val workerConfig = SerialWorkerConfig() + val cleanup = AttachmentCleanupService( + journalRepository = journal, + objectStorageService = storage, + attachmentRepository = attachments, + config = workerConfig + ) + + val jobs = ExposedAttachmentJobRepository(attachments) + val broker = application.get() val file = withContext(Dispatchers.IO) { Files.createTempFile("cleanup-test-", ".png") @@ -58,28 +138,78 @@ class AttachmentCleanupTest { } var orphanId: UUID? = null - val failingStorage = object : ObjectStorageRepository by storage { - override suspend fun put(id: UUID, file: File, contentType: String) { - storage.put(id, file, contentType) + val failingStorage = object : ObjectStorageService by storage { + override suspend fun putStaging( + id: UUID, + file: File, + contentType: String + ) { + storage.putStaging( + id = id, + file = file, + contentType = contentType + ) + orphanId = id error("injected failure after PUT, before metadata COMMIT") } } + try { - val failingService = DefaultAttachmentService(journal, attachments, ExposedChatRepository(), failingStorage) + val failingService = DefaultAttachmentService( + journalRepository = journal, + attachmentRepository = attachments, + chatRepository = ExposedChatRepository(), + objectStorageService = failingStorage, + jobRepository = jobs, + jobBrokerService = broker + ) assertFailsWith { - failingService.upload(chat.id, user.id, "file.png", "image/png", file) + failingService.upload( + chatId = chat.id, + actorId = user.id, + filename = "file.png", + contentType = "image/png", + file = file + ) } val id = requireNotNull(orphanId) - assertEquals(null, suspendedTransaction { attachments.findById(id) }) - assertContentEquals(file.readBytes(), storage.get(id)) - cleanup.cleanup() - assertContentEquals(file.readBytes(), storage.get(id), "fresh upload must survive grace") - val service = DefaultAttachmentService(journal, attachments, ExposedChatRepository(), storage) - val published = service.upload(chat.id, user.id, "ready.png", "image/png", file) + assertEquals(null, suspendedTransaction { attachments.findById(id) }) + assertContentEquals(file.readBytes(), storage.getStaging(id)) + + cleanup.cleanup() + + assertContentEquals( + file.readBytes(), + storage.getStaging(id), + "fresh upload must survive grace" + ) + + val service = DefaultAttachmentService( + journalRepository = journal, + attachmentRepository = attachments, + chatRepository = ExposedChatRepository(), + objectStorageService = storage, + jobRepository = jobs, + jobBrokerService = broker + ) + val published = service.upload( + chatId = chat.id, + actorId = user.id, + filename = "ready.png", + contentType = "image/png", + file = file + ) + + withTimeout(30.seconds) { + while (suspendedTransaction { attachments.findById(published.id)?.status } != "ready") { + delay(50.milliseconds) + } + } + suspendedTransaction { UploadJournalModel.update({ UploadJournalModel.id eq id }) { it[expiresAt] = CurrentTimestamp + longLiteral(-1) @@ -87,14 +217,22 @@ class AttachmentCleanupTest { } } - val unavailableStorage = object : ObjectStorageRepository by storage { + val unavailableStorage = object : ObjectStorageService by storage { override suspend fun delete(id: UUID) { error("injected S3 delete outage") } } - assertEquals(0, AttachmentCleanupService(journal, unavailableStorage).cleanup()) - assertContentEquals(file.readBytes(), storage.get(id)) + val unavailableCleanup = AttachmentCleanupService( + journalRepository = journal, + objectStorageService = unavailableStorage, + attachmentRepository = attachments, + config = workerConfig + ) + + assertEquals(0, unavailableCleanup.cleanup()) + assertContentEquals(file.readBytes(), storage.getStaging(id)) + suspendedTransaction { UploadJournalModel.update({ UploadJournalModel.id eq id }) { it[nextAttemptAt] = CurrentTimestamp + longLiteral(-1) @@ -102,12 +240,12 @@ class AttachmentCleanupTest { } assertTrue(cleanup.cleanup() >= 1) - assertFailsWith { storage.get(id) } + assertFailsWith { storage.getStaging(id) } assertFalse(suspendedTransaction { journal.lockForCompletion(id) }) assertContentEquals(file.readBytes(), storage.get(published.id)) // A process suspended longer than grace can complete a late PUT, but cannot publish metadata. - storage.put(id, file, "image/png") + storage.putStaging(id, file, "image/png") suspendedTransaction { UploadJournalModel.update({ UploadJournalModel.id eq id }) { it[nextAttemptAt] = CurrentTimestamp + longLiteral(-1) @@ -115,7 +253,7 @@ class AttachmentCleanupTest { } assertTrue(cleanup.cleanup() >= 1) - assertFailsWith { storage.get(id) } + assertFailsWith { storage.getStaging(id) } assertContentEquals(file.readBytes(), storage.get(published.id)) } finally { file.delete() diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt new file mode 100644 index 00000000..241d2653 --- /dev/null +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/AttachmentPipelineTest.kt @@ -0,0 +1,471 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.mtusi.messenger.main.config.worker.SerialWorkerConfig +import com.mtusi.messenger.main.infrastructure.controllers.message.requests.CreateMessageRequest +import com.mtusi.messenger.main.infrastructure.dto.Attachment +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.infrastructure.dto.Message +import com.mtusi.messenger.main.infrastructure.dto.ProcessedImage +import com.mtusi.messenger.main.infrastructure.models.attachment.AttachmentJobModel +import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.AttachmentJobRepository +import com.mtusi.messenger.main.infrastructure.repositories.attachment.jobs.JobClaim +import com.mtusi.messenger.main.infrastructure.services.attachment.jobs.JobBrokerService +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.ObjectStorageService +import com.mtusi.messenger.main.infrastructure.types.ApiException +import com.mtusi.messenger.main.infrastructure.types.ErrorCode +import com.mtusi.messenger.main.plugins.suspendedTransaction +import com.mtusi.messenger.main.support.auth +import com.mtusi.messenger.main.support.createChat +import com.mtusi.messenger.main.support.idempotencyKey +import com.mtusi.messenger.main.support.register +import com.mtusi.messenger.main.support.runApp +import com.mtusi.messenger.main.utils.computeSha256 +import com.mtusi.messenger.main.utils.exposed.operations.plus +import io.ktor.client.call.body +import io.ktor.client.request.get +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import java.awt.image.BufferedImage +import java.io.ByteArrayOutputStream +import java.util.UUID +import java.util.concurrent.atomic.AtomicInteger +import javax.imageio.ImageIO +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.longLiteral +import org.jetbrains.exposed.v1.datetime.CurrentTimestamp +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import org.koin.ktor.ext.get + +class AttachmentPipelineTest { + + @Test + fun `stopped worker leaves a durable queued attachment and redelivery preserves its result`() = runApp(startWorker = false) { client -> + + val alice = client.register("pipeline") + val bob = client.register("reader") + val chat = client.createChat( + creator = alice, + title = "pipeline", + memberIds = listOf(alice.id, bob.id) + ) + val original = image() + + val upload = client.post("/api/v1/chats/${chat.id}/attachments?filename=sample.png") { + auth(alice) + contentType(ContentType.Image.PNG) + setBody(original) + } + + assertEquals(HttpStatusCode.Accepted, upload.status) + + val queued = upload.body() + + assertEquals("queued", queued.status) + assertEquals(emptyList(), queued.variants) + assertEquals(null, queued.errorCode) + assertEquals(original.computeSha256(), queued.sha256) + + val path = "/api/v1/attachments/${queued.id}" + val metadata = client.get(path) { auth(bob) }.body() + val unavailableOriginal = client.get("$path/content") { auth(bob) } + val unavailableThumbnail = client.get("$path/content?variant=thumbnail") { auth(bob) } + + assertEquals("queued", metadata.status) + assertEquals(HttpStatusCode.Conflict, unavailableOriginal.status) + assertEquals(HttpStatusCode.Conflict, unavailableThumbnail.status) + + val sent = client.post("/api/v1/chats/${chat.id}/messages") { + auth(alice) + idempotencyKey() + contentType(ContentType.Application.Json) + setBody( + CreateMessageRequest( + text = "", + attachmentIds = listOf(queued.id.toString()) + ) + ) + } + + assertEquals(HttpStatusCode.Created, sent.status) + + val job = jobFor(queued.id) + val processing = application.get() + + assertTrue(processing.process(job)) + + val ready = client.get(path) { auth(bob) }.body() + + assertEquals("ready", ready.status) + assertEquals(null, ready.errorCode) + + val variant = ready.variants.single() + + assertEquals("thumbnail", variant.name) + assertEquals("image/jpeg", variant.contentType) + assertEquals(256, variant.width) + assertEquals(160, variant.height) + + val thumbnail = client.get("$path/content?variant=thumbnail") { auth(bob) }.body() + + assertEquals(variant.sizeBytes, thumbnail.size.toLong()) + assertEquals(variant.sha256, thumbnail.computeSha256()) + + val decoded = withContext(Dispatchers.IO) { + thumbnail.inputStream().use { input -> + ImageIO.read(input) + } + } + val downloaded = client.get("$path/content") { auth(bob) }.body() + + assertEquals(256, decoded.width) + assertEquals(160, decoded.height) + assertContentEquals(original, downloaded) + + assertTrue( + processing.process(job), + "a ready redelivery must be acknowledged without another effect" + ) + + val replayedMetadata = client.get(path) { auth(bob) }.body() + val replayedThumbnail = client.get("$path/content?variant=thumbnail") { auth(bob) }.body() + val attempts = suspendedTransaction { + AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.id eq job.jobId) + .single()[AttachmentJobModel.attempts] + } + + assertEquals(ready, replayedMetadata) + assertContentEquals(thumbnail, replayedThumbnail) + assertEquals(1, attempts) + } + + @Test + fun `attempts are bounded and failed attachments reject content and message links`() = runApp(startWorker = false) { client -> + + val user = client.register("attempts") + val chat = client.createChat( + creator = user, + title = "attempts", + memberIds = listOf(user.id) + ) + + val queued = client.post("/api/v1/chats/${chat.id}/attachments?filename=sample.png") { + auth(user) + contentType(ContentType.Image.PNG) + setBody(image()) + }.body() + + val path = "/api/v1/attachments/${queued.id}" + val key = UUID.randomUUID().toString() + + suspend fun send(key: String) = client.post("/api/v1/chats/${chat.id}/messages") { + auth(user) + idempotencyKey(key) + contentType(ContentType.Application.Json) + setBody( + CreateMessageRequest( + text = "", + attachmentIds = listOf(queued.id.toString()) + ) + ) + } + + val sent = send(key) + + assertEquals(HttpStatusCode.Created, sent.status) + + val message = sent.body() + val job = jobFor(queued.id) + val jobs = application.get() + val firstToken = UUID.randomUUID() + + val firstClaim = suspendedTransaction { + jobs.claim( + job = job, + token = firstToken, + maxAttempts = 2, + leaseSeconds = 60 + ) + } + val processing = client.get(path) { auth(user) }.body() + val competingClaim = suspendedTransaction { + jobs.claim( + job = job, + token = UUID.randomUUID(), + maxAttempts = 2, + leaseSeconds = 60 + ) + } + + assertEquals(JobClaim.Acquired(1), firstClaim) + assertEquals("processing", processing.status) + assertEquals(JobClaim.Busy, competingClaim) + + val terminalFirstAttempt = suspendedTransaction { + jobs.fail( + job = job, + token = firstToken, + errorCode = "storage_unavailable", + permanent = false, + maxAttempts = 2 + ) + } + val pendingRetry = client.get(path) { auth(user) }.body() + + assertFalse(terminalFirstAttempt) + assertEquals("queued", pendingRetry.status) + assertNull(pendingRetry.errorCode) + + val pendingJob = suspendedTransaction { + AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.id eq job.jobId) + .single() + } + + assertEquals("storage_unavailable", pendingJob[AttachmentJobModel.lastErrorCode]) + assertNotNull(pendingJob[AttachmentJobModel.nextAttemptAt]) + + val retry = job.copy(attempt = 2) + val secondToken = UUID.randomUUID() + val staleClaim = suspendedTransaction { + jobs.claim( + job = job, + token = UUID.randomUUID(), + maxAttempts = 2, + leaseSeconds = 60 + ) + } + val retryClaim = suspendedTransaction { + jobs.claim( + job = retry, + token = secondToken, + maxAttempts = 2, + leaseSeconds = 60 + ) + } + + assertEquals(JobClaim.Retry(2), staleClaim) + assertEquals(JobClaim.Acquired(2), retryClaim) + + val claimedJob = suspendedTransaction { + AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.id eq job.jobId) + .single() + } + + assertNull(claimedJob[AttachmentJobModel.nextAttemptAt]) + + val terminalRetry = suspendedTransaction { + jobs.fail( + job = retry, + token = secondToken, + errorCode = "retry_exhausted", + permanent = false, + maxAttempts = 2 + ) + } + + assertTrue(terminalRetry) + + val failed = client.get(path) { auth(user) }.body() + + assertEquals("failed", failed.status) + assertEquals("retry_exhausted", failed.errorCode) + assertEquals(emptyList(), failed.variants) + + val replay = send(key) + + assertEquals(HttpStatusCode.Created, replay.status) + assertEquals(message, replay.body(), "failure after sending must not invalidate idempotent replay") + + val unavailableContent = client.get("$path/content") { auth(user) } + val rejectedMessage = send(UUID.randomUUID().toString()) + + assertEquals(HttpStatusCode.Conflict, unavailableContent.status) + assertEquals(HttpStatusCode.Conflict, rejectedMessage.status) + + assertTrue(application.get().process(job)) + assertEquals(failed, client.get(path) { auth(user) }.body()) + } + + @Test + fun `busy redelivery lets the next attachment finish before lease expiration`() = runApp(startWorker = false) { client -> + + val user = client.register("busyredelivery") + val chat = client.createChat( + creator = user, + title = "busy redelivery", + memberIds = listOf(user.id) + ) + + suspend fun upload(name: String): Attachment = + client.post("/api/v1/chats/${chat.id}/attachments?filename=$name.png") { + auth(user) + contentType(ContentType.Image.PNG) + setBody(image()) + }.body() + + val busy = upload("busy") + val busyJob = jobFor(busy.id) + val jobs = application.get() + + val claim = suspendedTransaction { + jobs.claim( + job = busyJob, + token = UUID.randomUUID(), + maxAttempts = 5, + leaseSeconds = 60 + ) + } + + assertEquals(JobClaim.Acquired(1), claim) + + val next = upload("next") + val broker = application.get() + val worker = application.get() + + suspend fun metadata(attachment: Attachment): Attachment { + return client.get("/api/v1/attachments/${attachment.id}") { auth(user) } + .body() + } + + suspend fun awaitReady(attachment: Attachment) { + withTimeout(15.seconds) { + while (metadata(attachment).status != "ready") { + delay(50.milliseconds) + } + } + } + + coroutineScope { + val consumer = launch { + broker.consume(worker::process) + } + + try { + awaitReady(next) + assertEquals("processing", metadata(busy).status) + + suspendedTransaction { + AttachmentJobModel.update(where = { AttachmentJobModel.id eq busyJob.jobId }) { + it[leaseUntil] = CurrentTimestamp + longLiteral(-1) + } + } + + awaitReady(busy) + } finally { + consumer.cancelAndJoin() + } + } + } + + @Test + fun `worker distinguishes permanent image errors from bounded transient failures`() = runApp(startWorker = false) { client -> + + val user = client.register("workererrors") + val chat = client.createChat( + creator = user, + title = "errors", + memberIds = listOf(user.id) + ) + + for (permanent in listOf(false, true)) { + val queued = client.post("/api/v1/chats/${chat.id}/attachments?filename=sample.png") { + auth(user) + contentType(ContentType.Image.PNG) + setBody(image()) + }.body() + + val path = "/api/v1/attachments/${queued.id}" + val job = jobFor(queued.id) + val calls = AtomicInteger() + + val processor = object : ImageProcessingService { + override suspend fun thumbnail( + bytes: ByteArray, + declaredType: String + ): ProcessedImage { + calls.incrementAndGet() + + if (permanent) { + throw ApiException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "invalid image") + } + + error("injected temporary processing failure") + } + } + + val service = DefaultAttachmentProcessingService( + jobRepository = application.get(), + attachmentRepository = application.get(), + objectStorageService = application.get(), + imageProcessingService = processor, + jobBrokerService = application.get(), + config = SerialWorkerConfig(maxAttempts = 2) + ) + + assertTrue(service.process(job)) + + if (!permanent) { + val pendingRetry = client.get(path) { auth(user) }.body() + + assertEquals("queued", pendingRetry.status) + assertTrue(service.process(job.copy(attempt = 2))) + } + + val failed = client.get(path) { auth(user) }.body() + val expectedError = if (permanent) "invalid_image" else "retry_exhausted" + val expectedCalls = if (permanent) 1 else 2 + + assertEquals("failed", failed.status) + assertEquals(expectedError, failed.errorCode) + assertEquals(expectedCalls, calls.get()) + + assertTrue(service.process(job)) + assertEquals(expectedCalls, calls.get(), "terminal redelivery must not run the processor again") + } + } + + private suspend fun jobFor(attachmentId: UUID): AttachmentJob = suspendedTransaction { + val row = AttachmentJobModel + .selectAll() + .where(AttachmentJobModel.attachmentId eq attachmentId) + .single() + + AttachmentJob( + jobId = row[AttachmentJobModel.id].value, + attachmentId = attachmentId, + createdAt = row[AttachmentJobModel.createdAt], + pipelineVersion = row[AttachmentJobModel.pipelineVersion] + ) + } + + private fun image(): ByteArray = ByteArrayOutputStream().use { output -> + ImageIO.write(BufferedImage(320, 200, BufferedImage.TYPE_INT_RGB), "png", output) + output.toByteArray() + } +} diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt new file mode 100644 index 00000000..65783a11 --- /dev/null +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/ImageProcessingServiceTest.kt @@ -0,0 +1,284 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.drew.imaging.jpeg.JpegMetadataReader +import com.drew.metadata.exif.ExifIFD0Directory +import com.drew.metadata.exif.ExifReader +import com.mtusi.messenger.main.infrastructure.types.ApiException +import com.mtusi.messenger.main.infrastructure.types.ErrorCode +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import java.awt.Color +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.zip.CRC32 +import javax.imageio.ImageIO +import kotlin.math.abs +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class ImageProcessingServiceTest { + + private val service = DefaultImageProcessingService() + private val colors = listOf(Color.RED, Color.GREEN, Color.BLUE, Color.YELLOW) + + @Test + fun `thumbnail fits the box floors dimensions and never enlarges the source`() = runBlocking { + + val cases = listOf( + Triple(320 to 200, 256 to 160, "png"), + Triple(200 to 320, 160 to 256, "jpeg"), + Triple(501 to 300, 256 to 153, "png"), + Triple(63 to 41, 63 to 41, "jpeg"), + Triple(600 to 1, 256 to 1, "png"), + Triple(1 to 600, 1 to 256, "png") + ) + + for ((size, expected, format) in cases) { + val source = BufferedImage(size.first, size.second, BufferedImage.TYPE_INT_RGB) + val bytes = encode(source, format) + val original = bytes.copyOf() + + val thumbnail = service.thumbnail(bytes, "image/$format") + val decoded = decode(thumbnail.bytes) + + assertEquals(expected.first, thumbnail.width) + assertEquals(expected.second, thumbnail.height) + assertEquals(expected.first, decoded.width) + assertEquals(expected.second, decoded.height) + assertEquals("image/jpeg", thumbnail.contentType) + assertEquals(0xff, thumbnail.bytes[0].toInt() and 0xff) + assertEquals(0xd8, thumbnail.bytes[1].toInt() and 0xff) + assertContentEquals(original, bytes, "thumbnail generation must not modify the original") + } + } + + @Test + fun `transparent and semitransparent pixels are composited on white`() = runBlocking { + + val source = BufferedImage(80, 40, BufferedImage.TYPE_INT_ARGB) + val graphics = source.createGraphics() + + try { + graphics.color = Color(255, 0, 0, 128) + graphics.fillRect(0, 0, 40, 40) + } finally { + graphics.dispose() + } + + val bytes = encode(source, "png") + val thumbnail = service.thumbnail(bytes, "image/png") + val decoded = decode(thumbnail.bytes) + + assertColor(Color.WHITE, Color(decoded.getRGB(60, 20))) + assertColor(Color(255, 127, 127), Color(decoded.getRGB(20, 20))) + assertFalse(decoded.colorModel.hasAlpha()) + } + + @Test + fun `all JPEG EXIF orientations rotate or mirror pixels and strip orientation from the thumbnail`() = runBlocking { + + val encoded = encode(quadrants(), "jpeg") + val expectedCorners = listOf( + listOf(0, 1, 2, 3), + listOf(1, 0, 3, 2), + listOf(3, 2, 1, 0), + listOf(2, 3, 0, 1), + listOf(0, 2, 1, 3), + listOf(2, 0, 3, 1), + listOf(3, 1, 2, 0), + listOf(1, 3, 0, 2) + ) + + for (orientation in 1..8) { + val bytes = withJpegExif(encoded, orientation) + val original = bytes.copyOf() + val thumbnail = service.thumbnail(bytes, "image/jpeg") + val decoded = decode(thumbnail.bytes) + + assertEquals(if (orientation >= 5) 40 else 80, decoded.width) + assertEquals(if (orientation >= 5) 80 else 40, decoded.height) + assertCorners(decoded, expectedCorners[orientation - 1]) + assertContentEquals(original, bytes) + + val metadata = ByteArrayInputStream(thumbnail.bytes).use { + JpegMetadataReader.readMetadata(it, listOf(ExifReader())) + } + + assertNull(metadata.getFirstDirectoryOfType(ExifIFD0Directory::class.java)) + } + } + + @Test + fun `PNG EXIF orientation is applied before resizing`() = runBlocking { + + val source = BufferedImage(320, 200, BufferedImage.TYPE_INT_RGB) + val bytes = withPngExif(encode(source, "png"), 6) + val thumbnail = service.thumbnail(bytes, "image/png") + + assertEquals(160, thumbnail.width) + assertEquals(256, thumbnail.height) + + val orientedQuadrants = withPngExif(encode(quadrants(), "png"), 7) + val small = service.thumbnail(orientedQuadrants, "image/png") + val decoded = decode(small.bytes) + + assertCorners(decoded, listOf(3, 1, 2, 0)) + } + + @Test + fun `missing or unknown EXIF orientation preserves the normal image`() = runBlocking { + + val bytes = encode(quadrants(), "jpeg") + + for (original in listOf(bytes, withJpegExif(bytes, 9))) { + val thumbnail = service.thumbnail(original, "image/jpeg") + val decoded = decode(thumbnail.bytes) + + assertEquals(80, thumbnail.width) + assertEquals(40, thumbnail.height) + assertCorners(decoded, listOf(0, 1, 2, 3)) + } + } + + @Test + fun `invalid images and MIME mismatches are rejected before publishing a thumbnail`() = runBlocking { + + val invalidImages = listOf( + "not an image".toByteArray() to "image/png", + encode(quadrants(), "png") to "image/jpeg", + encode(quadrants(), "jpeg").copyOf(40) to "image/jpeg" + ) + + for ((bytes, type) in invalidImages) { + val failure = assertFailsWith { + service.thumbnail(bytes, type) + } + + assertEquals(ErrorCode.UNSUPPORTED_MEDIA_TYPE, failure.code) + } + } + + private fun quadrants(): BufferedImage { + + val image = BufferedImage(80, 40, BufferedImage.TYPE_INT_RGB) + val graphics = image.createGraphics() + + try { + colors.forEachIndexed { index, color -> + graphics.color = color + graphics.fillRect((index % 2) * 40, (index / 2) * 20, 40, 20) + } + } finally { + graphics.dispose() + } + + return image + } + + private fun assertCorners( + image: BufferedImage, + expected: List + ) { + + expected.forEachIndexed { index, color -> + val x = if (index % 2 == 0) image.width / 4 else image.width * 3 / 4 + val y = if (index / 2 == 0) image.height / 4 else image.height * 3 / 4 + assertColor(colors[color], Color(image.getRGB(x, y))) + } + } + + private fun assertColor( + expected: Color, + actual: Color + ) { + assertTrue(abs(expected.red - actual.red) <= 12, "red: expected $expected, got $actual") + assertTrue(abs(expected.green - actual.green) <= 12, "green: expected $expected, got $actual") + assertTrue(abs(expected.blue - actual.blue) <= 12, "blue: expected $expected, got $actual") + } + + private fun encode( + image: BufferedImage, + format: String + ): ByteArray = ByteArrayOutputStream().use { output -> + check(ImageIO.write(image, format, output)) + output.toByteArray() + } + + private suspend fun decode( + bytes: ByteArray + ): BufferedImage = withContext(Dispatchers.IO) { + ByteArrayInputStream(bytes).use { input -> + ImageIO.read(input) + } + } + + private fun tiff(orientation: Int): ByteArray { + return ByteBuffer.allocate(26) + .order(ByteOrder.LITTLE_ENDIAN) + .put('I'.code.toByte()) + .put('I'.code.toByte()) + .putShort(42) + .putInt(8) + .putShort(1) + .putShort(0x0112) + .putShort(3) + .putInt(1) + .putShort(orientation.toShort()) + .putShort(0) + .putInt(0) + .array() + } + + private fun withJpegExif( + bytes: ByteArray, + orientation: Int + ): ByteArray { + + val exif = byteArrayOf(69, 120, 105, 102, 0, 0) + tiff(orientation) + return ByteArrayOutputStream().use { output -> + output.write(bytes, 0, 2) + DataOutputStream(output).apply { + writeShort(0xffe1) + writeShort(exif.size + 2) + write(exif) + write(bytes, 2, bytes.size - 2) + } + output.toByteArray() + } + } + + private fun withPngExif( + bytes: ByteArray, + orientation: Int + ): ByteArray { + + val type = "eXIf".toByteArray(Charsets.US_ASCII) + val data = tiff(orientation) + val crc = CRC32().apply { + update(type) + update(data) + } + + return ByteArrayOutputStream().use { output -> + output.write(bytes, 0, 33) + DataOutputStream(output).apply { + writeInt(data.size) + write(type) + write(data) + writeInt(crc.value.toInt()) + write(bytes, 33, bytes.size - 33) + } + output.toByteArray() + } + } +} diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt new file mode 100644 index 00000000..ca127059 --- /dev/null +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/LegacyAttachmentMigrationTest.kt @@ -0,0 +1,129 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment + +import com.mtusi.messenger.main.config.storage.SerialStorageConfig +import com.mtusi.messenger.main.infrastructure.repositories.attachment.AttachmentRepository +import com.mtusi.messenger.main.infrastructure.services.attachment.storage.S3ObjectStorageService +import com.mtusi.messenger.main.plugins.suspendedTransaction +import com.mtusi.messenger.main.support.TestDatabase +import com.mtusi.messenger.main.support.TestStorage +import com.mtusi.messenger.main.support.runApp +import com.mtusi.messenger.main.utils.computeSha256 +import io.ktor.client.request.get +import io.ktor.http.HttpStatusCode +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.flywaydb.core.Flyway +import org.koin.ktor.ext.get +import org.testcontainers.containers.PostgreSQLContainer +import java.awt.image.BufferedImage +import java.io.ByteArrayOutputStream +import java.sql.DriverManager +import java.util.UUID +import javax.imageio.ImageIO +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.milliseconds + +class LegacyAttachmentMigrationTest { + + @Test + fun `migration schedules a thumbnail for an existing lab4 original`() { + + val id = UUID.randomUUID() + val original = ByteArrayOutputStream().use { output -> + ImageIO.write(BufferedImage(320, 200, BufferedImage.TYPE_INT_RGB), "png", output) + output.toByteArray() + } + + TestDatabase.create().use { database -> + database.start() + TestDatabase.awaitConnectable(database) + seedLab4Attachment(database, id, original) + + val storage = S3ObjectStorageService( + SerialStorageConfig( + endpoint = TestStorage.endpoint, + bucket = TestStorage.BUCKET, + accessKey = TestStorage.ACCESS_KEY, + secretKey = TestStorage.SECRET_KEY + ) + ) + + runBlocking { + storage.put( + id = id, + bytes = original, + contentType = "image/png" + ) + } + + runApp(database = database) { client -> + assertEquals(HttpStatusCode.OK, client.get("/health/live").status) + + val attachments = application.get() + val ready = withTimeout(30_000.milliseconds) { + var attachment = suspendedTransaction { checkNotNull(attachments.findById(id)) } + + while (attachment.status != "ready") { + check(attachment.status != "failed") { attachment.errorCode.orEmpty() } + delay(50.milliseconds) + attachment = suspendedTransaction { checkNotNull(attachments.findById(id)) } + } + + attachment + } + + val variant = ready.variants.single() + val thumbnail = storage.getThumbnail(id) + + assertContentEquals(original, storage.get(id)) + assertEquals("thumbnail", variant.name) + assertEquals(256, variant.width) + assertEquals(160, variant.height) + assertEquals(variant.sha256, thumbnail.computeSha256()) + } + } + } + + private fun seedLab4Attachment( + database: PostgreSQLContainer<*>, + id: UUID, + original: ByteArray + ) { + + Flyway.configure() + .dataSource(database.jdbcUrl, database.username, database.password) + .locations("classpath:migrations") + .target("5") + .load() + .migrate() + + DriverManager.getConnection(database.jdbcUrl, database.username, database.password).use { connection -> + connection.prepareStatement( + """ + WITH owner AS ( + INSERT INTO users VALUES (?, 'legacy', 'Legacy', 'unused', CURRENT_TIMESTAMP) + RETURNING id + ), chat AS ( + INSERT INTO chats VALUES (?, 'Legacy', CURRENT_TIMESTAMP) + RETURNING id + ) + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, created_at) + SELECT ?, chat.id, owner.id, 'legacy.png', 'image/png', ?, ?, CURRENT_TIMESTAMP + FROM owner CROSS JOIN chat + """.trimIndent() + ).use { insert -> + insert.setObject(1, UUID.randomUUID()) + insert.setObject(2, UUID.randomUUID()) + insert.setObject(3, id) + insert.setLong(4, original.size.toLong()) + insert.setString(5, original.computeSha256()) + assertEquals(1, insert.executeUpdate()) + } + } + } + +} diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/RabbitMqJobBrokerServiceTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/RabbitMqJobBrokerServiceTest.kt new file mode 100644 index 00000000..f496bad8 --- /dev/null +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/attachment/jobs/RabbitMqJobBrokerServiceTest.kt @@ -0,0 +1,137 @@ +package com.mtusi.messenger.main.infrastructure.services.attachment.jobs + +import com.mtusi.messenger.main.config.broker.SerialBrokerConfig +import com.mtusi.messenger.main.infrastructure.dto.AttachmentJob +import com.mtusi.messenger.main.support.TestBroker +import com.rabbitmq.client.ConnectionFactory +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import java.util.UUID +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.time.Clock +import kotlin.time.Duration.Companion.milliseconds + +class RabbitMqJobBrokerServiceTest { + + @Test + fun `confirmed job survives broker restart and negative acknowledgement preserves its identity`() = runBlocking { + + val config = TestBroker.config().copy( + connectionTimeoutMs = 1000, + operationTimeoutMs = 2000 + ) + val job = AttachmentJob( + jobId = UUID.randomUUID(), + attachmentId = UUID.randomUUID(), + createdAt = Clock.System.now() + ) + val marker = job.copy( + jobId = UUID.randomUUID(), + attachmentId = UUID.randomUUID() + ) + val publisher = RabbitMqJobBrokerService(config) + + try { + // No producer remains to republish after restart: the next delivery must + // come from the broker's persisted data. + publisher.publish(job) + } finally { + publisher.close() + } + + val recoveredConfig = withTimeout(30_000.milliseconds) { + val container = TestBroker.container + val containerId = container.containerId + + withContext(Dispatchers.IO) { + container.dockerClient.restartContainerCmd(containerId).exec() + } + + assertEquals(containerId, TestBroker.container.containerId) + + // Docker may assign another ephemeral host port when the container restarts. + val recoveredConfig = config.copy(port = TestBroker.port) + val broker = RabbitMqJobBrokerService(recoveredConfig) + + try { + val firstDelivery = CompletableDeferred() + val redelivery = CompletableDeferred() + val markerDelivery = CompletableDeferred() + + val consumer = launch { + broker.consume { delivered -> + when { + delivered.jobId == marker.jobId -> { + markerDelivery.complete(delivered) + awaitCancellation() + } + + firstDelivery.complete(delivered) -> false + + else -> { + redelivery.complete(delivered) + true + } + } + } + } + + try { + assertEquals(job, firstDelivery.await()) + assertEquals(job, redelivery.await()) + + // The marker synchronizes with the receive loop and stays unacknowledged. + broker.publish(marker) + assertEquals(marker, markerDelivery.await()) + } finally { + consumer.cancelAndJoin() + } + } finally { + broker.close() + } + + recoveredConfig + } + + // The closed channel must requeue the marker and only the marker. + assertEquals( + listOf(marker.jobId.toString()), + drainMessageIds(recoveredConfig) + ) + } + + private suspend fun drainMessageIds( + config: SerialBrokerConfig + ): List = withContext(Dispatchers.IO) { + + val factory = ConnectionFactory().apply { + host = config.host + port = config.port + username = config.username + password = requireNotNull(config.password) + connectionTimeout = config.connectionTimeoutMs + handshakeTimeout = config.operationTimeoutMs + channelRpcTimeout = config.operationTimeoutMs + } + + factory.newConnection().use { connection -> + connection.createChannel().use { channel -> + buildList { + while (true) { + val delivery = channel.basicGet(config.queue, true) ?: break + + add(delivery.props.messageId) + } + } + } + } + } + +} diff --git a/src/test/kotlin/com/mtusi/messenger/main/support/TestApp.kt b/src/test/kotlin/com/mtusi/messenger/main/support/TestApp.kt index 0ba00322..d630f0df 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/support/TestApp.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/support/TestApp.kt @@ -6,6 +6,7 @@ import com.mtusi.messenger.main.infrastructure.controllers.user.requests.CreateU import com.mtusi.messenger.main.infrastructure.dto.Chat import com.mtusi.messenger.main.infrastructure.dto.User import com.mtusi.messenger.main.module +import com.mtusi.messenger.main.plugins.configureAttachmentWorker import com.typesafe.config.ConfigFactory import io.ktor.client.HttpClient import io.ktor.client.call.body @@ -62,6 +63,12 @@ private fun testConfig( ) = HoconApplicationConfig( ConfigFactory.parseMap( mapOf( + "broker.host" to TestBroker.host, + "broker.port" to TestBroker.port, + "broker.username" to TestBroker.USERNAME, + "broker.password" to TestBroker.PASSWORD, + "broker.queue" to "test-${UUID.randomUUID()}", + "storage.endpoint" to TestStorage.endpoint, "storage.bucket" to TestStorage.BUCKET, "storage.accessKey" to TestStorage.ACCESS_KEY, @@ -100,19 +107,35 @@ fun runApp( redis: RedisEndpoint = TestRedis.endpoint(), idempotencyTtlSeconds: Long = 86400, sessionRevokeTimeoutMs: Long = 8000, + startWorker: Boolean = true, block: suspend ApplicationTestBuilder.(HttpClient) -> Unit ) = testApplication { + environment { - config = testConfig(sessionTtlSeconds, database, databaseConnectionTimeoutMs, redis, idempotencyTtlSeconds, sessionRevokeTimeoutMs) + config = testConfig( + sessionTtlSeconds = sessionTtlSeconds, + database = database, + databaseConnectionTimeoutMs = databaseConnectionTimeoutMs, + redis = redis, + idempotencyTtlSeconds = idempotencyTtlSeconds, + sessionRevokeTimeoutMs = sessionRevokeTimeoutMs + ) } + application { module() + + if (startWorker) { + configureAttachmentWorker() + } } + val client = createClient { install(ContentNegotiation) { json(testJson) } } + block(client) } diff --git a/src/test/kotlin/com/mtusi/messenger/main/support/TestBroker.kt b/src/test/kotlin/com/mtusi/messenger/main/support/TestBroker.kt new file mode 100644 index 00000000..0652e4dc --- /dev/null +++ b/src/test/kotlin/com/mtusi/messenger/main/support/TestBroker.kt @@ -0,0 +1,52 @@ +package com.mtusi.messenger.main.support + +import com.github.dockerjava.api.model.ExposedPort +import com.mtusi.messenger.main.config.broker.SerialBrokerConfig +import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait +import java.util.UUID + +object TestBroker { + + const val USERNAME = "test-worker" + const val PASSWORD = "test-broker-password" + + val container by lazy { + GenericContainer("rabbitmq@sha256:cd4009416f83d7e5895020cd71b0963f3b8a4dfc3de74bbb6fb3bfe5f81f01e0") + .withEnv("RABBITMQ_DEFAULT_USER", USERNAME) + .withEnv("RABBITMQ_DEFAULT_PASS", PASSWORD) + .withExposedPorts(5672) + .waitingFor(Wait.forLogMessage(".*Server startup complete.*\\n", 1)) + .also { + it.start() + } + } + + val host: String + get() = container.host + + val port: Int + get() { + val bindings = container.dockerClient + .inspectContainerCmd(container.containerId) + .exec() + .networkSettings + .ports + .bindings + + return bindings.getValue(ExposedPort.tcp(5672)) + .first() + .hostPortSpec + .toInt() + } + + fun config(queue: String = "attachments-test-${UUID.randomUUID()}"): SerialBrokerConfig { + return SerialBrokerConfig( + host = host, + port = port, + username = USERNAME, + password = PASSWORD, + queue = queue + ) + } +}