diff --git a/.dockerignore b/.dockerignore
index bc16e55..19e41e7 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
+
+.lab5-faults
+bin
diff --git a/.env.example b/.env.example
index ecbfeb2..b0567a9 100644
--- a/.env.example
+++ b/.env.example
@@ -30,9 +30,11 @@ RECONCILE_GRACE_PERIOD=1h
# SeaweedFS image and persistent bucket bootstrap are pinned in compose.yaml.
# API has four upload slots and one decoder per replica, GOMEMLIMIT=384MiB.
-# Planned lab5 async pipeline settings; the ported lab4 application does not use these yet.
-BROKER_URL_FILE=/run/secrets/broker_url
+# Async lab5 pipeline. Broker credentials are generated by bootstrap.
+BROKER_PASSWORD=replace-with-local-broker-password
WORKER_CONCURRENCY=2
+WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
-JOB_TIMEOUT_SECONDS=60
-STAGING_RETENTION_SECONDS=86400
+JOB_TIMEOUT=60s
+JOB_LEASE=90s
+STAGING_RETENTION=24h
diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml
new file mode 100644
index 0000000..0ae3427
--- /dev/null
+++ b/.github/workflows/application.yml
@@ -0,0 +1,28 @@
+name: Application
+on: [push, pull_request]
+permissions:
+ contents: read
+jobs:
+ application:
+ runs-on: ubuntu-latest
+ timeout-minutes: 25
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v5
+ with:
+ go-version: '1.27.2'
+ cache-dependency-path: service/go.sum
+ - run: make check build vet
+ - run: make up
+ - run: make test test-go test-storage test-images test-staging test-broker-persistence
+ - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
+ - run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
+ - run: make restore
+ - run: make test-pipeline
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: pipeline-evidence
+ path: evidence/pipeline.json
+ - run: make down
+ if: always()
diff --git a/.gitignore b/.gitignore
index 73a022b..01540a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,5 @@ coverage/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
/.ignore/
+
+.lab5-faults/
diff --git a/Dockerfile b/Dockerfile
index 38980df..a953e59 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,6 +16,9 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0
+RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0
+COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml
+
COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger
ENV APP_HOST=0.0.0.0 \
diff --git a/Makefile b/Makefile
index db8c42a..55b3c45 100644
--- a/Makefile
+++ b/Makefile
@@ -13,8 +13,8 @@ OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen
.PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt
help:
- @echo "up/down: lab5 stand (ported lab4 application); build/vet: Go checks"
- @echo "test: lab5 API smoke (requires async implementation); test-go/test-storage: existing Go checks"
+ @echo "up/down: lab5 async image pipeline; build/vet: Go checks"
+ @echo "test: lab5 API smoke; test-go/test-storage: Go checks"
@echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic"
@echo "check/validate: lab5 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks"
@echo "reconcile/reconcile-apply: storage consistency"
@@ -111,3 +111,18 @@ reconcile:
reconcile-apply:
./scripts/compose.sh run --rm -T operator storage reconcile --apply
+
+.PHONY: test-pipeline
+test-pipeline:
+ $(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json
+
+.PHONY: test-images test-staging
+test-images:
+ $(PYTHON) student-tests/images.py
+
+test-staging:
+ $(PYTHON) student-tests/storage.py
+
+.PHONY: test-broker-persistence
+test-broker-persistence:
+ $(PYTHON) student-tests/broker_persistence.py
diff --git a/REPORT.md b/REPORT.md
index e98f75e..489af1e 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,37 +1,173 @@
# Отчёт — лабораторная 5
-- Автор, группа: TODO
-- Целевая оценка; выбранный трек 5, если нужен: TODO
-- Commit/PR: TODO после публикации
-- Использованные источники, библиотеки и помощь ИИ: TODO
+- Автор, группа: не предоставлены владельцем проекта.
+- Целевая оценка: **5**, трек 2 — доказанная идемпотентность versioned pipeline.
+- Ревизия: базовый commit и состояние рабочего дерева записываются в `evidence/pipeline.json`; решение пока не закоммичено.
+- Использованы Go, pgx, AWS SDK, RabbitMQ `amqp091-go`, ImageMagick;
## Запуск из чистого clone
-TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
+Требуются Docker Compose и для локальных Go-проверок Go 1.27+. Контейнерная сборка использует Go 1.27.2. Образы закреплены версиями/digest в `compose.yaml` и `Dockerfile`; ImageMagick и модули JPEG/WebP — 7.1.2.35-r0.
-## Архитектура и решения
+```sh
+make check build vet
+make up
+curl --fail http://localhost:18085/health/ready
+make test
+make test-go
+make test-storage
+make test-images test-staging test-broker-persistence
+make test-persistence ACTION_SCRIPT=scripts/restart.sh
+make test-failover ACTION_SCRIPT=scripts/stop-one.sh
+make restore
+make test-pipeline
+make down
+```
-TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
+Полная валидация схем:
+
+```sh
+python3 -m venv .venv
+.venv/bin/python -m pip install -r requirements-dev.txt
+make validate PYTHON=.venv/bin/python
+```
+
+Bootstrap сохраняет существующие локальные настройки и генерирует синтетические credentials PostgreSQL, Redis, S3 и RabbitMQ. `.env.lab5` и `secrets/lab5` игнорируются Git. Миграции выполняются отдельным одноразовым процессом, bucket создаётся идемпотентно. Внешний endpoint — только `127.0.0.1:18085`; внутренние хранилища портов на host не публикуют. API-реплик по умолчанию две.
+
+`make down` сохраняет volumes. **Намеренное удаление учебных данных**: `./scripts/compose.sh down --volumes`; эта команда не входит в обычные проверки и автоматически не выполняется.
+
+## Архитектура и границы
+
+```mermaid
+flowchart LR
+ Client --> LB[HAProxy]
+ LB --> API[API x2]
+ API --> Staging[Private S3 staging]
+ API --> DB[PostgreSQL: metadata + jobs + outbox]
+ DB --> Dispatcher
+ Dispatcher --> Broker[RabbitMQ durable queue]
+ Broker --> Worker
+ Worker --> Results[Private S3 versioned results]
+ Worker --> DB
+```
+
+Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Attachment service принимает DTO и reader, проверяет доступ и целостность изображения, управляет транзакцией и состояниями. Репозитории выполняют операции хранения; HTTP, retry-политик и преобразования изображений в SQL/триггерах нет. Канонизация тела сообщения, сравнение идемпотентных запросов и создание Message перенесены в chat service.
+
+До `202` весь проверенный оригинал уже записан в S3 staging, а metadata, job и outbox — в одной завершённой транзакции PostgreSQL. При неопределённом COMMIT нет слепого компенсирующего DELETE. Ответ содержит сохранённый снимок `queued`, даже если быстрый worker уже начал работу.
+
+Staging находится в том же S3-сервисе, что конечные объекты: это разделение prefix и жизненных циклов, а не обещание отложить всю сетевую передачу файла. Проверка корректности изображения при upload включает ограниченное декодирование; resize/crop/WebP и публикация конечного оригинала выполняются отдельно в worker.
+
+API readiness проверяет PostgreSQL и S3. Недоступность broker/worker не блокирует транзакционный приём через outbox. Worker и dispatcher переподключаются после отказа зависимостей. Redis остаётся необязательным кэшем, очередь в него не помещается.
+
+### Инфраструктура
+
+| Роль → service | Версия | Проверка | Volume / сеть / credentials |
+| --- | --- | --- | --- |
+| API → api, ≥2 | собственная сборка Go 1.27.2 | `/health/ready` | без локального durable state; default + storage; signing/S3/broker files, DB env |
+| PostgreSQL → postgres | 17.11, digest в Compose | pg_isready | postgres_data; default; локальный пароль env |
+| Кэш → redis | 8.2.11, digest в Compose | authenticated PING | redis_data, AOF; default; локальный пароль env |
+| LB → lb | HAProxy 3.2, digest в Compose | backend readiness | без volume данных; default; публичный localhost порт |
+| S3 → s3 | SeaweedFS 4.52, digest в Compose | cluster status; bootstrap проверяет запись | s3_data; internal storage; credentials JSON file |
+| Broker → broker | RabbitMQ 4.2.0, digest в Compose | rabbitmq-diagnostics check_running | broker_data; default; синтетические credentials |
+| Worker → worker | образ приложения + ImageMagick | процесс/reconnect, состояние jobs | эфемерные ограниченные temp files; default + storage; secret files |
+| Dispatcher → dispatcher | образ приложения | процесс/reconnect, состояние outbox | без локального durable state; default; broker secret file |
+
+У worker/dispatcher отключён унаследованный HTTP healthcheck: HTTP-сервер они не запускают. Состояние `running` само по себе не подтверждает прогресс; проверять нужно jobs/outbox и end-to-end сценарий.
+
+### Доставка и повторы
+
+- Durable RabbitMQ queue, persistent messages, publisher confirms и mandatory routing. Имя broker фиксировано, чтобы новый контейнер открывал ту же persistent node database; readiness ждёт запуска RabbitMQ application, а не только Erlang node. Consumer ack выполняется после устойчивого результата, постановки следующей попытки либо записи terminal failed.
+- Сообщения соответствуют `jobs.schema.json`: UUID, версия, attempt, timestamp, опциональный trace carrier. Байтов файла, URL и credentials в них нет. Worker читает доверенные storage metadata из PostgreSQL.
+- `job_id` стабилен для вложения и версии. Redelivery сама не увеличивает attempt; управляемый retry/recovery увеличивает его. По умолчанию пять попыток, exponential backoff с jitter 0.5–30 секунд в зависимости от номера попытки.
+- Publisher имеет 20-секундный deadline на самом socket (одного контекста PublishWithContext недостаточно для ограничения сетевой записи). Dispatcher захватывает одну запись с lease 30 секунд, публикует её и фиксирует результат с проверкой token. Просроченный dispatcher не может изменить запись нового владельца. После пяти неудачных публикаций остаётся dead_letter-запись и наблюдаемый failed job.
+- Worker: concurrency=2, prefetch=2, timeout=60s, lease=90s. Lease recovery работает независимо от consumer. Неактивный lease не продлевается: работа дольше лимита повторяется ограниченно.
+- Постоянная ошибка заканчивается немедленно (`invalid_image`, `source_missing`, `source_corrupt`); временная повторяется и после исчерпания бюджета даёт `retry_exhausted`. API получает только безопасный error_code.
+- Реестр failed — PostgreSQL. Replay явный, только для текущего failed job с доступным source; записывается аудит, номер attempt продолжает расти, выдаётся новый бюджет из пяти попыток. Уже успешную job replay не запускает.
+
+```sh
+./scripts/compose.sh run --rm -T operator jobs list
+./scripts/compose.sh run --rm -T operator jobs replay --id JOB_UUID
+./scripts/compose.sh run --rm -T operator outbox replay --id JOB_UUID
+./scripts/compose.sh run --rm -T operator jobs reprocess --id ATTACHMENT_UUID --pipeline-version 2
+```
+
+`outbox replay` восстанавливает логическую failed job вместе с новой outbox-записью; ID аргумента — job, не отдельная доставка. Старые dead_letter-записи сохраняются для диагностики и не переводят новую попытку обратно в failed.
+
+### Версии и инварианты
+
+Manifest имеет PK `(attachment_id, pipeline_version, variant)`. Варианты публикуются одной DB-транзакцией только после записи всех объектов. Завершение проверяет текущую версию вложения, состояние job, token и действующий lease. Ready не откатывается из-за старой доставки.
+
+Объектные ключи включают вложение, версию, variant и SHA-256. Повтор одинакового эффекта использует тот же ключ; другая версия пишет в свой namespace. Поэтому устаревшая попытка не перезапишет новые байты S3, даже когда БД отклоняет её завершение. Незавершённые объекты возможны и удаляются как сироты после grace; они не становятся опубликованным вторым набором вариантов.
+
+Версия 1 использует WebP quality 80, версия 2 — 85. Остальные параметры одинаковы. Reprocess доступен для ready-вложения и строго большей поддерживаемой версии; на время новой работы статус queued и content=409. Сохраняются прежние manifest, новый оригинал побайтно совпадает с исходником.
+
+### Изображения и cleanup
+
+Thumbnail: JPEG, вписан в 256×256 без увеличения, размеры округляются вниз с минимумом 1 px. Crop: центральный квадрат JPEG 128×128 с разрешённым увеличением. Optimized: WebP внутри 1280×1280 без увеличения. Применяется EXIF orientation, alpha компонуется на белый, метаданные производных удаляются. JPEG quality=85. Не обещается уменьшение каждого файла.
+
+ImageMagick запускается без shell, отдельным дочерним процессом с отменой по timeout. Политика запрещает delegates/filters и разрешает только нужные coders; memory=256MiB, map=0, disk=256MiB, thread=1, time=60s, list-length=4. Worker ограничен 1 CPU/1GiB; его локальные файлы — только восстанавливаемые временные данные. API принимает максимум четыре одновременных upload и выполняет одну проверку декодирования на реплику. Пиксели проверяются до полной распаковки. Превышение ресурсов заканчивается ошибкой, а не бесконечным retry.
+
+Staging удаляется после устойчивого ready; failed source хранится 24h. Источники, которыми являются исторические опубликованные results, физически сохраняются. После удаления failed source replay отклоняется; оператор может восстановить оригинал с проверкой размера/SHA и затем явно повторить job. Reconcile защищает ссылки metadata/manifest и активные jobs, удаляет сироты только после grace=1h (>2×upload timeout). Старые originals lab4 переводятся миграцией на первую версию pipeline без изменения ID и ссылок сообщений.
## Доказательства критериев
-| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
-| --- | --- | --- | --- |
-| Публичный smoke | make test | TODO | TODO |
-| Остальные пункты уровня 3 | TODO | TODO | TODO |
-| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
-| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
+Фактический вывод команд сохраняется отдельно от описания реализации. `evidence/pipeline.json` содержит повторяемые сценарии, UUID, длительности и результат каждого запуска; `evidence/images.json` — проверки изображений. Проверки защиты staging, удаления после ready, retention failed и восстановления исходника — `evidence/storage.json`. Остальные результаты — `evidence/checks.txt`.
-TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
+| Критерий | Команда / наблюдение | Доказательство |
+| --- | --- | --- |
+| Структура, схемы, сборка и анализ | make check; make validate; make build; make vet | checks.txt |
+| Публичный контракт и предыдущие лабораторные | make test; make test-go; make test-storage | checks.txt |
+| Сессия и данные после перезапуска; отказ API | test-persistence / test-failover с действительными hooks | checks.txt |
+| 202 при остановленном worker, сообщение с queued-вложением, рестарт stores/broker и ready | worker_down_and_store_restart | pipeline.json |
+| Опубликованное сообщение переживает пересоздание broker с тем же volume при остановленном dispatcher | make test-broker-persistence; hostname=broker сохраняет имя узла | broker-persistence.json |
+| Thumbnail/crop/WebP, orientation, alpha, реальные размеры и hashes | student-tests/images.py | images.json |
+| Сбой до/после S3 side effect и до/после COMMIT результата, до ack | crash_before_objects / after_original_put / after_objects / after_result_commit / before_ack | pipeline.json |
+| Confirm получен, outbox ещё processing; повтор публикации безопасен | crash_confirm_before_outbox_mark | pipeline.json |
+| Ready redelivery — no-op | ready_redelivery, 3 копии | pipeline.json |
+| Реальный lease expiry, более новая попытка/версия, возобновление старого worker и поздняя старая доставка | lease_reorder_and_stale_version | pipeline.json |
+| Staging защищён при queued; после ready удалён; failed retention и restore проверены | make test-staging | storage.json |
+| Retry ограничен, failed наблюдаем, replay явный | bounded_retry_and_explicit_replay / permanent_error | pipeline.json |
+| Сбой после staging и разрыв COMMIT/publish | api_crash_after_staging / api_crash_after_upload_commit | pipeline.json |
-## Эксперименты и отказы
+## Эксперимент трека 2
-TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
+**Гипотеза:** атомарное сохранение намерения, durable доставка, fencing в БД и разделение версий/контента в S3 сохраняют принятые задания и предотвращают публикацию устаревшего эффекта при проверяемых crash/reorder/duplicate сценариях.
-| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
-| --- | --- | --- | --- |
-| TODO | TODO | TODO | TODO |
+Методика: `make test-pipeline` включает исключительно для эксперимента `compose.faults.yaml`. Директивы в игнорируемом `.lab5-faults/` адресуются по job/attachment ID и точке сбоя. Барьер отмечает достижение точки; crash завершает процесс с кодом 86, restart policy поднимает его снова. Pause применяется один раз: другая попытка может завершиться после реального истечения lease. Общая директория не содержит токенов и очищается при завершении.
-## Ограничения и следующие шаги
+В эксперименте timeout=5s, lease=8s, concurrency/prefetch=2; остальные параметры не ускоряются искусственным изменением lease в БД. Окно после publisher confirm использует обычный outbox lease=30s. Для гарантированного повторного publish тест повторно делает свою outbox-запись pending; это внесение дубля, а не штатная операция пользователя. Для COMMIT/publish dispatcher заранее остановлен, проверяется pending-запись. Для crash до COMMIT отсутствие `202` не трактуется как потеря принятой работы.
-TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
+Каждый сценарий выполняется три раза с новыми синтетическими пользователями/вложениями. Основной файл — публичный PNG fixture 320×200, 111817 bytes; дополнительных параллельных upload в fault-эксперименте нет. Измеряется длительность сценария (включает Docker stop/start и проверки), а не чистая latency обработки. Точность таймеров — monotonic clock клиента; нагрузочный максимум этим экспериментом не измеряется.
+
+Окружение выполнения: Apple M4 Pro, 14 CPU, host RAM 24GiB; Docker Desktop arm64, 14 CPU и около 7.75GiB RAM. CPU/RAM контейнеров ограничены Compose. Исследование проводится на одном Docker-host с сохранением volumes, без симуляции потери диска.
+
+
+### Наблюдавшиеся результаты
+
+Все **39/39** сценариев трека завершились PASS (13 сценариев × 3 повтора). Интервалы ниже включают действия Docker и проверки; это не измерение чистого processing latency.
+
+| Сценарий | Минимум–максимум, с |
+| --- | --- |
+| worker_down_and_store_restart | 63.864–66.563 |
+| crash_before_objects | 10.888–12.269 |
+| crash_after_original_put | 11.222–12.171 |
+| crash_after_objects | 11.174–11.340 |
+| crash_after_result_commit | 1.973–2.088 |
+| crash_before_ack | 2.017–2.098 |
+| ready_redelivery | 2.435–3.002 |
+| crash_confirm_before_outbox_mark | 33.988–36.021 |
+| lease_reorder_and_stale_version | 14.652–15.225 |
+| bounded_retry_and_explicit_replay | 13.737–15.238 |
+| permanent_error | 2.552–2.699 |
+| api_crash_after_staging | 0.144–0.817 |
+| api_crash_after_upload_commit | 2.141–2.328 |
+
+После эксперимента отдельная проверка retention обнаружила и позволила исправить гонку восстановления исходника: Restore обновляет возраст source, а Cleanup берёт тот же advisory lock. Эта корректировка проверена `make test-staging`. После добавления socket deadline повторно выполнены confirm/crash, lease + stale version и COMMIT/publish на итоговом образе: **3/3 PASS**, см. `evidence/pipeline-final.json`. Image ID исходного полного fault-прогона сохранён в pipeline.json, итогового — в pipeline-final.json.
+
+## Ограничения
+
+At-least-once доставка сохраняется: publish/confirm/COMMIT/ack не образуют одну распределённую транзакцию. Доказывается идемпотентность опубликованного эффекта в перечисленных сценариях, а не exactly-once доставка и не отсутствие любых возможных отказов.
+
+Один PostgreSQL, S3, broker, LB и host — отдельные точки отказа. Durable очередь здесь не означает кластер RabbitMQ. Невосстановимая потеря volumes не покрывается; backup/restore инфраструктуры не реализован. После окончательной ошибки нужен операторский replay; при длительном отказе брокера принятые задания могут стать наблюдаемыми failed. Накопление outbox/jobs не ограничено глобальной квотой: ограничены upload, worker concurrency/prefetch и ресурсы, но отдельный трек перегрузки не заявляется.
+
+Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acks](https://www.rabbitmq.com/docs/confirms), [Go client](https://github.com/rabbitmq/amqp091-go), [ImageMagick resource policy](https://imagemagick.org/resources/).
diff --git a/compose.faults.yaml b/compose.faults.yaml
new file mode 100644
index 0000000..75f47d8
--- /dev/null
+++ b/compose.faults.yaml
@@ -0,0 +1,22 @@
+# Explicit isolated experiment profile. Never enabled by make up.
+services:
+ api:
+ environment:
+ LAB5_FAULTS: "1"
+ volumes:
+ - ./.lab5-faults:/faults
+ worker:
+ environment:
+ LAB5_FAULTS: "1"
+ JOB_TIMEOUT: 5s
+ JOB_LEASE: 8s
+ volumes:
+ - ./.lab5-faults:/faults
+ dispatcher:
+ environment:
+ LAB5_FAULTS: "1"
+ volumes:
+ - ./.lab5-faults:/faults
+ operator:
+ environment:
+ LAB5_FAULTS: "1"
diff --git a/compose.yaml b/compose.yaml
index 217cce6..71bc0ca 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -40,6 +40,13 @@ x-api: &api
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret
+ BROKER_URL_FILE: /run/secrets/broker_url
+ WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
+ WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
+ JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
+ JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
+ JOB_LEASE: ${JOB_LEASE:-90s}
+ STAGING_RETENTION: ${STAGING_RETENTION:-24h}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s}
@@ -47,6 +54,7 @@ x-api: &api
GOMEMLIMIT: 384MiB
networks: [default, storage]
volumes:
+ - ./secrets/lab5/broker_url:/run/secrets/broker_url:ro
- ./secrets/lab5/private_key.pem:/run/certs/private_key.pem:ro
- ./secrets/lab5/public_key.pem:/run/certs/public_key.pem:ro
- ./secrets/lab5/s3_api_access:/run/secrets/s3_access:ro
@@ -78,6 +86,50 @@ services:
depends_on:
postgres:
condition: service_healthy
+ broker:
+ hostname: broker
+ image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524
+ restart: unless-stopped
+ cpus: 0.5
+ mem_limit: 512m
+ environment:
+ RABBITMQ_DEFAULT_USER: messenger
+ RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh}
+ volumes:
+ - broker_data:/var/lib/rabbitmq
+ healthcheck:
+ test: [CMD, rabbitmq-diagnostics, -q, check_running]
+ interval: 3s
+ timeout: 5s
+ retries: 30
+ stop_grace_period: 30s
+ worker:
+ <<: *api
+ command: [worker]
+ cpus: 1
+ mem_limit: 1g
+ stop_grace_period: 10s
+ healthcheck:
+ disable: true
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ s3-init:
+ condition: service_completed_successfully
+ broker:
+ condition: service_healthy
+ dispatcher:
+ <<: *api
+ command: [dispatcher]
+ cpus: 0.25
+ mem_limit: 128m
+ healthcheck:
+ disable: true
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ broker:
+ condition: service_healthy
lb:
image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0}
restart: unless-stopped
@@ -168,6 +220,7 @@ services:
- ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro
- ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro
- ./tests/fixtures:/fixtures:ro
+ - ./secrets/lab5/broker_url:/run/secrets/broker_url:ro
healthcheck:
disable: true
depends_on: {}
@@ -176,6 +229,7 @@ volumes:
postgres_data:
redis_data:
s3_data:
+ broker_data:
networks:
storage:
diff --git a/evidence/broker-persistence.json b/evidence/broker-persistence.json
new file mode 100644
index 0000000..84aa504
--- /dev/null
+++ b/evidence/broker-persistence.json
@@ -0,0 +1,10 @@
+{
+ "result": "PASS",
+ "attachment_id": "cb1e8982-980d-4286-8224-335b644a5fc5",
+ "broker_hostname": "broker",
+ "old_container_id": "16848707eae7bdfb195da43c912ad66399678cc0a48d100bea119619f7e62009",
+ "new_container_id": "0dfeb1f83d39adc4de9c7798d65b5f125927fe95ae050e4a17e8a5238713b81b",
+ "message_present_before_and_after": true,
+ "outbox_published_before_replacement": true,
+ "dispatcher_stopped_during_replacement": true
+}
diff --git a/evidence/build.json b/evidence/build.json
new file mode 100644
index 0000000..bdc85a1
--- /dev/null
+++ b/evidence/build.json
@@ -0,0 +1,14 @@
+{
+ "date_utc": "2026-11-13T09:29:53.527687+00:00",
+ "base_commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
+ "working_tree": "uncommitted solution",
+ "runtime_source_sha256": "cadfe8fc0d834a598d26e03d1c0348c907ac6fe9a4574aa959f156dfd69488d6",
+ "hash_method": "Sorted relative path + NUL + file content + NUL, for service/**/*.go, migrations/*.sql, go.mod, go.sum, Dockerfile and ImageMagick policy",
+ "image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f",
+ "go_local": "go version go1.27.0 darwin/arm64",
+ "docker_server": "29.7.2",
+ "compose": "Docker Compose version v5.3.1",
+ "docker_resources": "cpus=14 memory_bytes=8320954368 arch=aarch64",
+ "hardware": "Apple M4 Pro, 14 CPUs, 24 GiB host RAM",
+ "source_reuse": "Adapted core outbox from supplied mamont-optics project; original project left unchanged"
+}
diff --git a/evidence/checks.txt b/evidence/checks.txt
new file mode 100644
index 0000000..faa5a76
--- /dev/null
+++ b/evidence/checks.txt
@@ -0,0 +1,201 @@
+Lab5 verification, UTC 2026-11-13T09:29:53.527687+00:00
+All commands below completed with exit code 0.
+Go output excerpts omit SQL, runtime logs and synthetic request bodies.
+
+$ make check build vet; make validate PYTHON=.venv/bin/python
+python3 scripts/check_contract.py
+PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+cd service && go build ./...
+cd service && go build -o ../bin/benchmark ./cmd/benchmark
+cd service && go vet ./...
+.venv/bin/python scripts/validate_openapi.py
+PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+
+$ make test
+python3 tests/smoke.py --base-url "http://localhost:18085"
+PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+
+$ make test-go (race enabled)
+? gitea.privcloud.tech/mtusi2026/cmd/app [no test files]
+=== RUN TestLostPostResponseIsNotRetried
+--- PASS: TestLostPostResponseIsNotRetried (0.00s)
+PASS
+ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.461s
+? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files]
+=== RUN TestIdempotencyTransactionAndExpiry
+PASS
+ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.653s
+? gitea.privcloud.tech/mtusi2026/internal/config [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/dto [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files]
+=== RUN TestSessionFingerprints
+--- PASS: TestSessionFingerprints (0.02s)
+PASS
+ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.908s
+? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files]
+=== RUN TestVerifiedSessionIsRequestScoped
+--- PASS: TestVerifiedSessionIsRequestScoped (0.00s)
+=== RUN TestMissingTokenDoesNotCallSessionStore
+--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s)
+PASS
+ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 2.467s
+? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files]
+? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files]
+? gitea.privcloud.tech/mtusi2026/migrations [no test files]
+=== RUN TestInvalidJSON
+=== RUN TestInvalidJSON/malformed
+=== RUN TestInvalidJSON/wrong_type
+=== RUN TestInvalidJSON/unknown_field
+--- PASS: TestInvalidJSON (0.01s)
+ --- PASS: TestInvalidJSON/malformed (0.01s)
+ --- PASS: TestInvalidJSON/wrong_type (0.00s)
+ --- PASS: TestInvalidJSON/unknown_field (0.00s)
+=== RUN TestForeignChatIsHidden
+--- PASS: TestForeignChatIsHidden (0.54s)
+=== RUN TestEmptyMessage
+--- PASS: TestEmptyMessage (0.21s)
+=== RUN TestDuplicateUsername
+--- PASS: TestDuplicateUsername (0.28s)
+=== RUN TestMessagePagination
+--- PASS: TestMessagePagination (0.22s)
+=== RUN TestAttachmentContractAndErrors
+=== RUN TestAttachmentContractAndErrors/fake
+=== RUN TestAttachmentContractAndErrors/mime_mismatch
+=== RUN TestAttachmentContractAndErrors/wrong_type
+=== RUN TestAttachmentContractAndErrors/truncated
+--- PASS: TestAttachmentContractAndErrors (1.05s)
+ --- PASS: TestAttachmentContractAndErrors/fake (0.00s)
+ --- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s)
+ --- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s)
+ --- PASS: TestAttachmentContractAndErrors/truncated (0.00s)
+=== RUN TestConcurrentAttachmentMessage
+--- PASS: TestConcurrentAttachmentMessage (0.25s)
+=== RUN TestConcurrentMessages
+--- PASS: TestConcurrentMessages (1.84s)
+=== RUN TestIdempotencyContract
+--- PASS: TestIdempotencyContract (0.45s)
+=== RUN TestConcurrentIdempotencyAcrossReplicas
+--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s)
+PASS
+ok gitea.privcloud.tech/mtusi2026/tests 7.261s
+
+$ make test-storage (isolated database)
+=== RUN TestIdempotencyTransactionAndExpiry
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat
+=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object
+=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock
+--- PASS: TestIdempotencyTransactionAndExpiry (0.12s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.02s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s)
+ --- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s)
+PASS
+
+$ make test-images
+PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs
+
+$ make test-staging
+PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified
+
+$ make test-broker-persistence
+PASS: confirmed message survived broker container replacement with the same volume and stable node identity
+
+$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
+python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085"
+Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
+PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+
+$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
+python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085"
+Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
+{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.273}
+PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+
+Actual process state before persistence hook
+/mtusi2026lab5-api-1 pid=63585 started=2026-11-13T09:13:37.287792805Z
+/mtusi2026lab5-api-2 pid=63443 started=2026-11-13T09:13:37.185017014Z
+/mtusi2026lab5-broker-1 pid=16156 started=2026-11-13T09:06:10.180568751Z
+/mtusi2026lab5-dispatcher-1 pid=63444 started=2026-11-13T09:13:37.187137055Z
+/mtusi2026lab5-lb-1 pid=6588 started=2026-11-13T08:49:29.974451052Z
+/mtusi2026lab5-postgres-1 pid=15734 started=2026-11-13T09:06:09.241419334Z
+/mtusi2026lab5-redis-1 pid=15771 started=2026-11-13T09:06:09.289178667Z
+/mtusi2026lab5-s3-1 pid=17699 started=2026-11-13T09:06:34.353438762Z
+/mtusi2026lab5-worker-1 pid=66553 started=2026-11-13T09:14:11.339788001Z
+
+Actual process state after persistence hook
+/mtusi2026lab5-api-1 pid=73635 started=2026-11-13T09:15:52.601737132Z
+/mtusi2026lab5-api-2 pid=73510 started=2026-11-13T09:15:52.490649465Z
+/mtusi2026lab5-broker-1 pid=70234 started=2026-11-13T09:14:51.740755173Z
+/mtusi2026lab5-dispatcher-1 pid=73512 started=2026-11-13T09:15:52.489731548Z
+/mtusi2026lab5-lb-1 pid=73700 started=2026-11-13T09:15:52.696327173Z
+/mtusi2026lab5-postgres-1 pid=70113 started=2026-11-13T09:14:50.775761506Z
+/mtusi2026lab5-redis-1 pid=70044 started=2026-11-13T09:14:50.718397506Z
+/mtusi2026lab5-s3-1 pid=71476 started=2026-11-13T09:15:16.065232545Z
+/mtusi2026lab5-worker-1 pid=73511 started=2026-11-13T09:15:52.490188673Z
+
+Actual state after API failover, before restoring stopped replica
+api mtusi2026lab5-api-1 running healthy
+api mtusi2026lab5-api-2 exited
+broker mtusi2026lab5-broker-1 running healthy
+dispatcher mtusi2026lab5-dispatcher-1 running
+lb mtusi2026lab5-lb-1 running healthy
+migrate mtusi2026lab5-migrate-1 exited
+postgres mtusi2026lab5-postgres-1 running healthy
+redis mtusi2026lab5-redis-1 running healthy
+s3 mtusi2026lab5-s3-1 running healthy
+s3-init mtusi2026lab5-s3-init-1 exited
+worker mtusi2026lab5-worker-1 running
+
+Effective container resource limits
+/mtusi2026lab5-api-1 memory_bytes=536870912 cpu_nano=1000000000
+/mtusi2026lab5-api-2 memory_bytes=536870912 cpu_nano=1000000000
+/mtusi2026lab5-broker-1 memory_bytes=536870912 cpu_nano=500000000
+/mtusi2026lab5-dispatcher-1 memory_bytes=134217728 cpu_nano=250000000
+/mtusi2026lab5-lb-1 memory_bytes=134217728 cpu_nano=500000000
+/mtusi2026lab5-postgres-1 memory_bytes=1073741824 cpu_nano=2000000000
+/mtusi2026lab5-redis-1 memory_bytes=134217728 cpu_nano=250000000
+/mtusi2026lab5-s3-1 memory_bytes=536870912 cpu_nano=1000000000
+/mtusi2026lab5-worker-1 memory_bytes=1073741824 cpu_nano=1000000000
+
+make test-pipeline: 39/39 PASS; details and repetitions in pipeline.json
+Final image targeted fault checks: 3/3 PASS; details in pipeline-final.json
diff --git a/evidence/images.json b/evidence/images.json
new file mode 100644
index 0000000..717ac91
--- /dev/null
+++ b/evidence/images.json
@@ -0,0 +1,53 @@
+{
+ "results": [
+ {
+ "case": "small",
+ "result": "PASS",
+ "thumbnail": [
+ 16,
+ 8
+ ]
+ },
+ {
+ "case": "floor",
+ "result": "PASS",
+ "thumbnail": [
+ 256,
+ 76
+ ]
+ },
+ {
+ "case": "alpha",
+ "result": "PASS",
+ "thumbnail": [
+ 40,
+ 20
+ ]
+ },
+ {
+ "case": "orientation",
+ "result": "PASS",
+ "thumbnail": [
+ 128,
+ 256
+ ]
+ },
+ {
+ "case": "thin",
+ "result": "PASS",
+ "thumbnail": [
+ 1,
+ 256
+ ]
+ },
+ {
+ "case": "center_crop",
+ "result": "PASS",
+ "thumbnail": [
+ 256,
+ 85
+ ]
+ }
+ ],
+ "negative_cases": 4
+}
diff --git a/evidence/pipeline-final.json b/evidence/pipeline-final.json
new file mode 100644
index 0000000..e4293fd
--- /dev/null
+++ b/evidence/pipeline-final.json
@@ -0,0 +1,42 @@
+{
+ "date_utc": "2026-11-13T09:24:08.989020+00:00",
+ "commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
+ "working_tree": "implementation changes uncommitted",
+ "repeats": 1,
+ "worker": {
+ "concurrency": 2,
+ "prefetch": 2,
+ "timeout_seconds": 5,
+ "lease_seconds": 8,
+ "memory": "1GiB",
+ "cpus": 1
+ },
+ "results": [
+ {
+ "scenario": "crash_confirm_before_outbox_mark",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 33.527,
+ "job_id": "ce8a7658-c069-429a-b655-cb051f922e14"
+ },
+ {
+ "scenario": "lease_reorder_and_stale_version",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 14.623,
+ "attachment_id": "6e709809-9ca2-4fb8-b7f6-155a3aace89c",
+ "old_job_id": "421fa4dc-3704-4f87-a490-3110f062459a",
+ "pipeline_version": 2,
+ "manifest_rows": 8
+ },
+ {
+ "scenario": "api_crash_after_upload_commit",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 2.122,
+ "attachment_id": "293e92d6-4253-4fd9-8bad-719a488f6bdf",
+ "accepted_commit": true
+ }
+ ],
+ "image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f"
+}
diff --git a/evidence/pipeline.json b/evidence/pipeline.json
new file mode 100644
index 0000000..946326f
--- /dev/null
+++ b/evidence/pipeline.json
@@ -0,0 +1,337 @@
+{
+ "date_utc": "2026-11-13T09:09:05.756165+00:00",
+ "commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
+ "working_tree": "implementation changes uncommitted",
+ "repeats": 3,
+ "worker": {
+ "concurrency": 2,
+ "prefetch": 2,
+ "timeout_seconds": 5,
+ "lease_seconds": 8,
+ "memory": "1GiB",
+ "cpus": 1
+ },
+ "results": [
+ {
+ "scenario": "worker_down_and_store_restart",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 66.563,
+ "attachment_id": "912b15f8-2ddb-4cbc-8db7-526540256749",
+ "job_id": "4913bc34-1533-4140-a0dc-c53efa5be0dd",
+ "postgres_restarted": true
+ },
+ {
+ "scenario": "crash_before_objects",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 10.888,
+ "job_id": "bfdb496b-f2e0-4836-9559-1a9b07ed21c4",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_original_put",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 12.171,
+ "job_id": "8468e628-5eef-42c2-9a86-d050d74cb672",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_objects",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 11.174,
+ "job_id": "4c587b0f-fa79-4d9e-a3c7-4904db1e80d3",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_result_commit",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 1.973,
+ "job_id": "419c9eeb-6467-4138-b05e-92f1bc929df8",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_before_ack",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 2.017,
+ "job_id": "face4c1b-5597-4b24-8f61-a0e756e601de",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "ready_redelivery",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 2.435,
+ "job_id": "514c616d-367c-4693-8d33-60079b2ced87",
+ "copies": 3
+ },
+ {
+ "scenario": "crash_confirm_before_outbox_mark",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 33.996,
+ "job_id": "71f7c962-009e-4982-ae7b-e7f24fdcdc7f"
+ },
+ {
+ "scenario": "lease_reorder_and_stale_version",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 14.652,
+ "attachment_id": "52a2e241-60a1-4443-8ef7-2e8e3e860577",
+ "old_job_id": "1c5824ca-774e-4230-9764-3e0938f4efeb",
+ "pipeline_version": 2,
+ "manifest_rows": 8
+ },
+ {
+ "scenario": "bounded_retry_and_explicit_replay",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 14.47,
+ "job_id": "8b8f8975-f4c8-45cf-80e9-00626c4e5c36",
+ "failed_attempt": 5,
+ "replayed_attempt": 6
+ },
+ {
+ "scenario": "permanent_error",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 2.699,
+ "job_id": "e1948ce8-b194-4f66-8008-6e733ad42959"
+ },
+ {
+ "scenario": "api_crash_after_staging",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 0.817,
+ "attachment_id": "2d831ec0-5918-4290-8e26-3c2bec3cc0fd",
+ "accepted_commit": false
+ },
+ {
+ "scenario": "api_crash_after_upload_commit",
+ "run": 1,
+ "result": "PASS",
+ "seconds": 2.328,
+ "attachment_id": "f0da85c6-003c-4d79-ba01-f3f961064741",
+ "accepted_commit": true
+ },
+ {
+ "scenario": "worker_down_and_store_restart",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 66.013,
+ "attachment_id": "64517fe4-6419-4257-bbf5-a47a88b51068",
+ "job_id": "873b9510-7ada-4dc0-bdfb-3e204ae2bc3d",
+ "postgres_restarted": true
+ },
+ {
+ "scenario": "crash_before_objects",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 11.241,
+ "job_id": "1c0ea968-87d2-45df-bb9b-d8b45c68dc1d",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_original_put",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 11.222,
+ "job_id": "748017a5-8f09-4741-8e35-35e4dbcb3c55",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_objects",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 11.34,
+ "job_id": "489d2edd-b008-48fc-ba35-c9f64ee05ba2",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_result_commit",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 2.088,
+ "job_id": "0ed57d8a-3fcc-42a4-a31f-2f02ae4f5f5f",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_before_ack",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 2.026,
+ "job_id": "e9a8270c-3132-460a-a888-ee8722798e35",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "ready_redelivery",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 2.704,
+ "job_id": "b4f7d69b-3de5-4350-97bf-0e880ed94b0e",
+ "copies": 3
+ },
+ {
+ "scenario": "crash_confirm_before_outbox_mark",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 33.988,
+ "job_id": "5f7a89af-a045-423c-8c70-47b19615b577"
+ },
+ {
+ "scenario": "lease_reorder_and_stale_version",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 14.694,
+ "attachment_id": "a1230ca9-8001-4bb4-b648-51446f646cbf",
+ "old_job_id": "2f1e1fbf-856d-4605-873c-b23011072a4e",
+ "pipeline_version": 2,
+ "manifest_rows": 8
+ },
+ {
+ "scenario": "bounded_retry_and_explicit_replay",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 13.737,
+ "job_id": "737209e2-5d3c-4693-8394-fc8173b88ee2",
+ "failed_attempt": 5,
+ "replayed_attempt": 6
+ },
+ {
+ "scenario": "permanent_error",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 2.552,
+ "job_id": "e5e6dc90-9ea5-4d7c-aa11-c6c275d69793"
+ },
+ {
+ "scenario": "api_crash_after_staging",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 0.153,
+ "attachment_id": "ed8e2936-47fa-4717-9eb3-e63292c8a0ad",
+ "accepted_commit": false
+ },
+ {
+ "scenario": "api_crash_after_upload_commit",
+ "run": 2,
+ "result": "PASS",
+ "seconds": 2.159,
+ "attachment_id": "a739f237-2278-421c-aa55-6024e717ad89",
+ "accepted_commit": true
+ },
+ {
+ "scenario": "worker_down_and_store_restart",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 63.864,
+ "attachment_id": "95176e6d-301a-4725-9861-7133507184eb",
+ "job_id": "e5f8b9fa-e460-4441-92b0-2af2ac2540bf",
+ "postgres_restarted": true
+ },
+ {
+ "scenario": "crash_before_objects",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 12.269,
+ "job_id": "67c70b6e-ea42-4c1d-894c-d2234a70bf03",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_original_put",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 11.402,
+ "job_id": "386f9b0d-1b1f-42b6-95a0-a13d6b97f118",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_objects",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 11.259,
+ "job_id": "9c639085-f040-4989-90ba-54376c64630d",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_after_result_commit",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 2.068,
+ "job_id": "73fe51b1-32eb-4c93-b8b0-0ed920a6641d",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "crash_before_ack",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 2.098,
+ "job_id": "2947da70-b658-48c0-9aa2-905bc0927781",
+ "manifest_rows": 4
+ },
+ {
+ "scenario": "ready_redelivery",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 3.002,
+ "job_id": "4f67af5a-1014-4f91-9a68-54739f2edde7",
+ "copies": 3
+ },
+ {
+ "scenario": "crash_confirm_before_outbox_mark",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 36.021,
+ "job_id": "c4135c71-64e7-44ba-9e4f-bf8f7489a459"
+ },
+ {
+ "scenario": "lease_reorder_and_stale_version",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 15.225,
+ "attachment_id": "3bdce3c7-d7d5-49fb-9691-96e626317ae1",
+ "old_job_id": "dd3d35b5-14fe-420b-a650-e740bc43489c",
+ "pipeline_version": 2,
+ "manifest_rows": 8
+ },
+ {
+ "scenario": "bounded_retry_and_explicit_replay",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 15.238,
+ "job_id": "444a1bf1-42ed-4123-9ec7-7f614cd6d90f",
+ "failed_attempt": 5,
+ "replayed_attempt": 6
+ },
+ {
+ "scenario": "permanent_error",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 2.604,
+ "job_id": "d21ee0e0-702d-4459-8fd6-5e5f62c6c11d"
+ },
+ {
+ "scenario": "api_crash_after_staging",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 0.144,
+ "attachment_id": "963d2034-021f-492f-81f3-d31431a2bc0f",
+ "accepted_commit": false
+ },
+ {
+ "scenario": "api_crash_after_upload_commit",
+ "run": 3,
+ "result": "PASS",
+ "seconds": 2.141,
+ "attachment_id": "ea757ed8-98a8-4f3b-993e-556347380e26",
+ "accepted_commit": true
+ }
+ ],
+ "tested_image_id": "sha256:0b720b9bab11e4a762e8b180b26c290a43503c4db2db3436c201cb08725f24c0",
+ "runtime_source_sha256": "e6460140119a22471f992cd02c721b55463d0810a6a8e50333e193d674c7c79a",
+ "subsequent_fix": "Restore refreshes retention timestamp; Cleanup shares its source lock. Publisher has a physical socket deadline. Verified by storage.py and the additional final-image fault scenarios in pipeline-final.json; job processing, lease and manifest logic unchanged."
+}
diff --git a/evidence/storage.json b/evidence/storage.json
new file mode 100644
index 0000000..8a953a0
--- /dev/null
+++ b/evidence/storage.json
@@ -0,0 +1,24 @@
+{
+ "results": [
+ {
+ "scenario": "queued_staging_protected",
+ "result": "PASS",
+ "attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b"
+ },
+ {
+ "scenario": "staging_deleted_after_ready",
+ "result": "PASS",
+ "attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b"
+ },
+ {
+ "scenario": "failed_retention_and_missing_source_replay",
+ "result": "PASS",
+ "age_simulated_hours": 25
+ },
+ {
+ "scenario": "verified_restore_then_replay",
+ "result": "PASS",
+ "job_id": "8d6f4591-e10f-4616-b387-fe7ec5020a84"
+ }
+ ]
+}
diff --git a/infra/imagemagick-policy.xml b/infra/imagemagick-policy.xml
new file mode 100644
index 0000000..decc717
--- /dev/null
+++ b/infra/imagemagick-policy.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/scripts/bootstrap-broker.py b/scripts/bootstrap-broker.py
new file mode 100644
index 0000000..a1d7e47
--- /dev/null
+++ b/scripts/bootstrap-broker.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+"""Create synthetic local broker credentials, preserving existing settings."""
+from pathlib import Path
+import secrets
+from urllib.parse import quote
+root=Path(__file__).resolve().parents[1]
+env=root/'.env.lab5'
+values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#'))
+password=values.get('BROKER_PASSWORD')
+if not password:
+ password=secrets.token_hex(24)
+ with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n')
+p=root/'secrets/lab5/broker_url'
+if p.exists(): p.chmod(0o600)
+p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n')
+p.chmod(0o444)
+print('Local broker credentials ready.')
diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
index 4c09f78..f67bf84 100755
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -21,3 +21,5 @@ chmod 444 secrets/lab5/*.pem
printf '%s\n' 'Local lab5 settings and synthetic keys ready; existing files preserved.'
python3 scripts/bootstrap-storage.py
+
+python3 scripts/bootstrap-broker.py
diff --git a/scripts/restart.sh b/scripts/restart.sh
index 2469918..324c1ef 100755
--- a/scripts/restart.sh
+++ b/scripts/restart.sh
@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
-./scripts/compose.sh stop api lb
-./scripts/compose.sh restart postgres redis s3
+./scripts/compose.sh stop api lb worker dispatcher
+./scripts/compose.sh restart postgres redis s3 broker
./scripts/compose.sh run --rm -T s3-init
-./scripts/compose.sh up -d --wait --wait-timeout 90 api lb
+./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher
diff --git a/service/cmd/app/main.go b/service/cmd/app/main.go
index 804126d..353039d 100644
--- a/service/cmd/app/main.go
+++ b/service/cmd/app/main.go
@@ -27,6 +27,7 @@ import (
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/auth"
"gitea.privcloud.tech/mtusi2026/internal/service/chat"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"gitea.privcloud.tech/mtusi2026/internal/service/user"
"github.com/google/uuid"
stdhttp "net/http"
@@ -56,6 +57,9 @@ func run() error {
if err != nil {
return err
}
+ if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") {
+ return runPipeline(cfg, attachmentConfig, os.Args[1:])
+ }
if len(os.Args) > 1 && os.Args[1] == "storage" {
return runStorage(cfg, attachmentConfig, os.Args[2:])
}
@@ -91,6 +95,12 @@ func run() error {
chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache)
objectStore := objectstore.New(attachmentConfig)
attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc)
+ pipelineConfig, err := config.LoadPipeline()
+ if err != nil {
+ return err
+ }
+ outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil)
+ attachmentSvc.Configure(tm, outboxSvc, pipelineConfig)
httpRouter := router.New(
middleware.NewAuthMiddleware(authSvc),
diff --git a/service/cmd/app/pipeline.go b/service/cmd/app/pipeline.go
new file mode 100644
index 0000000..1cbabe4
--- /dev/null
+++ b/service/cmd/app/pipeline.go
@@ -0,0 +1,138 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "gitea.privcloud.tech/mtusi2026/internal/adapter/broker"
+ "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
+ "gitea.privcloud.tech/mtusi2026/internal/config"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
+ "gitea.privcloud.tech/mtusi2026/internal/service/attachment"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "log/slog"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+)
+
+func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error {
+ c, e := config.LoadPipeline()
+ if e != nil {
+ return e
+ }
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+ pool := postgres.New(cfg.Postgres)
+ defer pool.Close()
+ tm := txmanager.New(pool)
+ repo := &storage.AttachmentsRepo{Pool: pool}
+ rabbit := &broker.Rabbit{URL: c.BrokerURL}
+ ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit)
+ svc := attachment.New(repo, objectstore.New(a), a, nil)
+ svc.Configure(tm, ob, c)
+ ob.OnDead = svc.PublishFailed
+ if args[0] == "jobs" || args[0] == "outbox" {
+ if len(args) < 2 {
+ return errors.New("operator action required")
+ }
+ fs := flag.NewFlagSet(args[1], flag.ContinueOnError)
+ idText := fs.String("id", "", "job ID (attachment ID for reprocess)")
+ version := fs.Int("pipeline-version", 2, "new pipeline version")
+ if e = fs.Parse(args[2:]); e != nil {
+ return e
+ }
+ if args[1] == "list" {
+ jobs, e := repo.ListJobs(ctx)
+ if e != nil {
+ return e
+ }
+ return json.NewEncoder(os.Stdout).Encode(jobs)
+ }
+ id, e := uuid.Parse(*idText)
+ if e != nil {
+ return e
+ }
+ switch args[1] {
+ case "redeliver":
+ if os.Getenv("LAB5_FAULTS") != "1" {
+ return errors.New("redeliver is restricted to the explicit fault profile")
+ }
+ var payload []byte
+ e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ j, e := repo.JobTx(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ payload, e = json.Marshal(j.Message)
+ return e
+ })
+ if e == nil {
+ e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload})
+ }
+ case "replay":
+ e = svc.Replay(ctx, id)
+ case "reprocess":
+ e = svc.Reprocess(ctx, id, *version)
+ default:
+ return errors.New("unknown operator action")
+ }
+ if e != nil {
+ return e
+ }
+ return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id})
+ }
+ if c.BrokerURL == "" {
+ return errors.New("BROKER_URL_FILE is required")
+ }
+ if args[0] == "worker" {
+ // Recovery continues while the consumer is disconnected or a delivery is paused.
+ recoveryDone := make(chan struct{})
+ defer func() { stop(); <-recoveryDone }()
+ go func() {
+ defer close(recoveryDone)
+ tick := time.NewTicker(time.Second)
+ defer tick.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-tick.C:
+ if e := svc.Recover(ctx); e != nil {
+ slog.Warn("job_recovery_failed")
+ }
+ if e := svc.Cleanup(ctx); e != nil {
+ slog.Warn("source_cleanup_failed")
+ }
+ }
+ }
+ }()
+ for ctx.Err() == nil {
+ if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil {
+ slog.Warn("consumer_reconnecting")
+ }
+ select {
+ case <-ctx.Done():
+ case <-time.After(time.Second):
+ }
+ }
+ return nil
+ }
+ for ctx.Err() == nil {
+ _, e = ob.ProcessOnce(ctx)
+ if e != nil {
+ slog.Warn("dispatcher_retry")
+ }
+ select {
+ case <-ctx.Done():
+ case <-time.After(100 * time.Millisecond):
+ }
+ }
+ return nil
+}
diff --git a/service/cmd/app/storage.go b/service/cmd/app/storage.go
index daeb6a1..09d459c 100644
--- a/service/cmd/app/storage.go
+++ b/service/cmd/app/storage.go
@@ -20,7 +20,9 @@ import (
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
)
func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
@@ -93,6 +95,12 @@ func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
pool := postgres.New(cfg.Postgres)
defer pool.Close()
svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil)
+ jobs, e := config.LoadPipeline()
+ if e != nil {
+ return e
+ }
+ tm := txmanager.New(pool)
+ svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs)
switch args[0] {
case "reconcile":
fs := flag.NewFlagSet("reconcile", flag.ContinueOnError)
diff --git a/service/go.mod b/service/go.mod
index b816a85..ce30f28 100644
--- a/service/go.mod
+++ b/service/go.mod
@@ -52,6 +52,7 @@ require (
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect
+ github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
diff --git a/service/go.sum b/service/go.sum
index 9c564cf..d7b4c21 100644
--- a/service/go.sum
+++ b/service/go.sum
@@ -94,6 +94,8 @@ github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/Ge
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE=
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
+github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
diff --git a/service/internal/adapter/broker/rabbitmq.go b/service/internal/adapter/broker/rabbitmq.go
new file mode 100644
index 0000000..508a251
--- /dev/null
+++ b/service/internal/adapter/broker/rabbitmq.go
@@ -0,0 +1,170 @@
+package broker
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/fault"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
+ "github.com/google/uuid"
+ amqp "github.com/rabbitmq/amqp091-go"
+ "io"
+ "log/slog"
+ "net"
+ "regexp"
+ "time"
+)
+
+const Queue = "attachments.v1"
+
+var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
+
+type Rabbit struct{ URL string }
+
+func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) {
+ dial := amqp.DefaultDial(5 * time.Second)
+ if deadline > 0 {
+ dial = func(network, address string) (net.Conn, error) {
+ c, e := net.DialTimeout(network, address, 5*time.Second)
+ if e != nil {
+ return nil, e
+ }
+ if e = c.SetDeadline(time.Now().Add(deadline)); e != nil {
+ c.Close()
+ return nil, e
+ }
+ return c, nil
+ }
+ }
+ conn, e := amqp.DialConfig(r.URL, amqp.Config{Heartbeat: 10 * time.Second, Dial: dial})
+ if e != nil {
+ return nil, nil, errors.New("broker connection unavailable")
+ }
+ ch, e := conn.Channel()
+ if e != nil {
+ conn.Close()
+ return nil, nil, errors.New("broker channel unavailable")
+ }
+ _, e = ch.QueueDeclare(Queue+".invalid", true, false, false, false, nil)
+ if e == nil {
+ e = ch.ExchangeDeclare("attachments.invalid", "direct", true, false, false, false, nil)
+ }
+ if e == nil {
+ e = ch.QueueBind(Queue+".invalid", "invalid", "attachments.invalid", false, nil)
+ }
+ if e == nil {
+ _, e = ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{"x-dead-letter-exchange": "attachments.invalid", "x-dead-letter-routing-key": "invalid"})
+ }
+ if e != nil {
+ ch.Close()
+ conn.Close()
+ return nil, nil, errors.New("broker topology unavailable")
+ }
+ return conn, ch, nil
+}
+func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) error {
+ conn, ch, e := r.connect(20 * time.Second)
+ if e != nil {
+ return e
+ }
+ defer conn.Close()
+ defer ch.Close()
+ if e = ch.Confirm(false); e != nil {
+ return e
+ }
+ confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1))
+ returns := ch.NotifyReturn(make(chan amqp.Return, 1))
+ var job dto.JobMessage
+ if e = json.Unmarshal(m.Payload, &job); e != nil {
+ return e
+ }
+ fault.Hit(ctx, "before_publish", job.JobID)
+ if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: m.Payload}); e != nil {
+ return errors.New("broker publish failed")
+ }
+ select {
+ case c, ok := <-confirms:
+ if !ok || !c.Ack {
+ return errors.New("broker did not confirm")
+ }
+ select {
+ case <-returns:
+ return errors.New("unroutable job")
+ default:
+ }
+ case <-time.After(15 * time.Second):
+ return errors.New("confirm timeout")
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ fault.Hit(ctx, "after_publish", job.JobID)
+ return nil
+}
+func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error {
+ conn, ch, e := r.connect(0)
+ if e != nil {
+ return e
+ }
+ defer conn.Close()
+ defer ch.Close()
+ if e = ch.Qos(prefetch, 0, false); e != nil {
+ return e
+ }
+ deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil)
+ if e != nil {
+ return e
+ }
+ done := make(chan struct{})
+ go func() {
+ select {
+ case <-ctx.Done():
+ conn.Close()
+ case <-done:
+ }
+ }()
+ defer close(done)
+ tasks := make(chan amqp.Delivery)
+ finished := make(chan struct{}, concurrency)
+ for range concurrency {
+ go func() {
+ defer func() { finished <- struct{}{} }()
+ for d := range tasks {
+ var m dto.JobMessage
+ dec := json.NewDecoder(bytes.NewReader(d.Body))
+ dec.DisallowUnknownFields()
+ e := dec.Decode(&m)
+ if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) {
+ _ = d.Reject(false)
+ continue
+ }
+ if e = process(ctx, m); e != nil {
+ slog.Warn("job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable")
+ select {
+ case <-ctx.Done():
+ case <-time.After(200 * time.Millisecond):
+ }
+ _ = d.Nack(false, true)
+ continue
+ }
+ fault.Hit(ctx, "before_ack", m.JobID)
+ _ = d.Ack(false)
+ }
+ }()
+ }
+ for d := range deliveries {
+ select {
+ case tasks <- d:
+ case <-ctx.Done():
+ }
+ }
+ close(tasks)
+ for range concurrency {
+ <-finished
+ }
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ return errors.New("broker consumer disconnected")
+}
diff --git a/service/internal/adapter/controller/http/attachmentHandler.go b/service/internal/adapter/controller/http/attachmentHandler.go
index 577723c..d286d53 100644
--- a/service/internal/adapter/controller/http/attachmentHandler.go
+++ b/service/internal/adapter/controller/http/attachmentHandler.go
@@ -1,12 +1,14 @@
package http
import (
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
"gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider"
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework"
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"io"
+ "mime"
"net/http"
"strconv"
"strings"
@@ -41,7 +43,20 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename"))
return
}
- a, e := h.svc.Upload(w, r, user.UserID, id, values[0])
+ if !attachment.ValidFilename(values[0]) {
+ framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename"))
+ return
+ }
+ mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
+ if e != nil || (mt != "image/png" && mt != "image/jpeg") {
+ framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG"))
+ return
+ }
+ if r.ContentLength > h.svc.Config.MaxBytes {
+ framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large"))
+ return
+ }
+ a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)})
if e != nil {
if apperr.CodeFromError(e) == apperr.CodeTooManyRequests {
w.Header().Set("Retry-After", "1")
@@ -49,7 +64,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
framework.JSONError(r.Context(), w, e)
return
}
- framework.JSONResponse(w, http.StatusCreated, a)
+ framework.JSONResponse(w, http.StatusAccepted, a)
}
func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
user, e := framework.ParseToken(r, h.provider)
@@ -61,27 +76,32 @@ func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
if e != nil {
return
}
- a, e := h.svc.Get(r.Context(), user.UserID, id)
+
+ if !strings.HasSuffix(r.URL.Path, "/content") {
+ a, e := h.svc.Get(r.Context(), user.UserID, id)
+ if e != nil {
+ framework.JSONError(r.Context(), w, e)
+ return
+ }
+ framework.JSONResponse(w, 200, a.Public)
+ return
+ }
+ v := r.URL.Query().Get("variant")
+ if v == "" {
+ v = "original"
+ }
+ if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" {
+ framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant"))
+ return
+ }
+ obj, e := h.svc.Content(r.Context(), user.UserID, id, v)
if e != nil {
framework.JSONError(r.Context(), w, e)
return
}
- if !strings.HasSuffix(r.URL.Path, "/content") {
- framework.JSONResponse(w, 200, a.Public)
- return
- }
- if v := r.URL.Query().Get("variant"); v != "" && v != "original" {
- framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant"))
- return
- }
- obj, e := h.svc.Store.Get(r.Context(), a.Key)
- if e != nil {
- framework.JSONError(r.Context(), w, attachment.Unavailable())
- return
- }
defer obj.Body.Close()
- w.Header().Set("Content-Type", string(a.Public.ContentType))
- w.Header().Set("Content-Length", strconv.Itoa(a.Public.SizeBytes))
+ w.Header().Set("Content-Type", obj.ContentType)
+ w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
// Once streaming has started a transport failure terminates the response; no JSON
// error can be appended to an image. Content-Length exposes incomplete downloads.
diff --git a/service/internal/adapter/repository/storage/attachment_scan.go b/service/internal/adapter/repository/storage/attachment_scan.go
index 15fa213..5a10537 100644
--- a/service/internal/adapter/repository/storage/attachment_scan.go
+++ b/service/internal/adapter/repository/storage/attachment_scan.go
@@ -25,10 +25,15 @@ func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*Attachm
}
func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) {
var found bool
- e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)", key).Scan(&found)
+ e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found)
return found, e
}
func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) {
tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing)
return tag.RowsAffected() == 1, e
}
+
+func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error {
+ _, e := c.Exec(ctx, "SELECT 1")
+ return e
+}
diff --git a/service/internal/adapter/repository/storage/attachments.go b/service/internal/adapter/repository/storage/attachments.go
index 6d35c74..0a3b2bb 100644
--- a/service/internal/adapter/repository/storage/attachments.go
+++ b/service/internal/adapter/repository/storage/attachments.go
@@ -8,40 +8,34 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
- "gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
+ "encoding/json"
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
)
-type AttachmentRecord struct {
- Public contracts.Attachment
- Key string
- Missing bool
-}
+type AttachmentRecord = dto.AttachmentRecord
type AttachmentsRepo struct{ Pool *pgxpool.Pool }
-const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing"
+const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys"
func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) {
a := new(AttachmentRecord)
p := &a.Public
- if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing); err != nil {
+ var variants, keys []byte
+ if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil {
return nil, apperr.WrapPostgres(err, "attachment unavailable")
}
- p.Status = "ready"
- p.Variants = []contracts.Variant{}
+ if err := json.Unmarshal(variants, &p.Variants); err != nil {
+ return nil, err
+ }
+ if err := json.Unmarshal(keys, &a.VariantKeys); err != nil {
+ return nil, err
+ }
return a, nil
}
func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) {
return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id))
}
-func (r *AttachmentsRepo) Save(ctx context.Context, conn *pgxpool.Conn, a *AttachmentRecord) error {
- p := a.Public
- _, e := conn.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.Key)
- if e != nil {
- return apperr.WrapPostgres(e, "failed to save attachment")
- }
- return nil
-}
// Session locks span S3 work and the database commit. A lost connection releases
// the lock; the mandatory grace period also exceeds the maximum upload lifetime.
diff --git a/service/internal/adapter/repository/storage/idempotency_test.go b/service/internal/adapter/repository/storage/idempotency_test.go
index 2d151e2..db5164c 100644
--- a/service/internal/adapter/repository/storage/idempotency_test.go
+++ b/service/internal/adapter/repository/storage/idempotency_test.go
@@ -58,7 +58,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil {
+ if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil {
t.Fatal(err)
}
if err = tx.Rollback(ctx); err != nil {
@@ -75,7 +75,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- first, err := repo.CreateIdempotentTx(ctx, tx, req)
+ first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req)
if err != nil {
t.Fatal(err)
}
@@ -91,7 +91,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- second, err := repo.CreateIdempotentTx(ctx, tx, req)
+ second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req)
if err != nil {
t.Fatal(err)
}
@@ -115,7 +115,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if _, err = repo.CreateIdempotentTx(ctx, tx, req); err == nil {
+ if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil {
t.Fatal("expected injected failure")
}
if err = tx.Rollback(ctx); err != nil {
@@ -134,7 +134,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil {
+ if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil {
t.Fatal("transient failure was cached", err)
}
if err = tx.Commit(ctx); err != nil {
@@ -257,7 +257,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- _, err = repo.CreateIdempotentTx(ctx, tx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}})
+ _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}})
if err == nil {
t.Fatal("expected missing attachment FK failure")
}
diff --git a/service/internal/adapter/repository/storage/jobs.go b/service/internal/adapter/repository/storage/jobs.go
new file mode 100644
index 0000000..4bb9648
--- /dev/null
+++ b/service/internal/adapter/repository/storage/jobs.go
@@ -0,0 +1,134 @@
+package storage
+
+import (
+ "context"
+ "encoding/json"
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "time"
+)
+
+func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) }
+func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) {
+ return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id))
+}
+func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
+ p := a.Public
+ _, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status)
+ return e
+}
+func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
+ variants, e := json.Marshal(a.Public.Variants)
+ if e != nil {
+ return e
+ }
+ keys, e := json.Marshal(a.VariantKeys)
+ if e != nil {
+ return e
+ }
+ _, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing)
+ return e
+}
+
+const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at`
+
+func scanJob(row pgx.Row) (*dto.AttachmentJob, error) {
+ j := new(dto.AttachmentJob)
+ m := &j.Message
+ err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt)
+ m.SchemaVersion = 1
+ m.Type = "attachment.process.v1"
+ return j, err
+}
+func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) {
+ return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id))
+}
+func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
+ m := j.Message
+ _, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt)
+ return e
+}
+func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
+ _, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt)
+ return e
+}
+func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) {
+ rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until>'attempt')::integer=j.attempt LIMIT 100`)
+ if e != nil {
+ return nil, e
+ }
+ defer rows.Close()
+ var ids []uuid.UUID
+ for rows.Next() {
+ var id uuid.UUID
+ if e = rows.Scan(&id); e != nil {
+ return nil, e
+ }
+ ids = append(ids, id)
+ }
+ return ids, rows.Err()
+}
+func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) {
+ rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before)
+ if e != nil {
+ return nil, e
+ }
+ defer rows.Close()
+ var out []*dto.AttachmentRecord
+ for rows.Next() {
+ a, e := ScanAttachment(rows)
+ if e != nil {
+ return nil, e
+ }
+ out = append(out, a)
+ }
+ return out, rows.Err()
+}
+
+func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error {
+ _, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size)
+ return e
+}
+
+func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) {
+ var dead bool
+ e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead)
+ return dead, e
+}
diff --git a/service/internal/adapter/repository/storage/messages.go b/service/internal/adapter/repository/storage/messages.go
index 61c29ba..0c1b6f9 100644
--- a/service/internal/adapter/repository/storage/messages.go
+++ b/service/internal/adapter/repository/storage/messages.go
@@ -1,10 +1,8 @@
package storage
import (
- "bytes"
"context"
- "crypto/sha256"
- "encoding/json"
+
"errors"
"time"
@@ -26,55 +24,49 @@ func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo {
return &MessagesRepo{pool: pool}
}
-// CreateIdempotentTx serializes only requests sharing an idempotency scope.
-// The transaction commits both the message and its replay record, or neither.
-func (r *MessagesRepo) CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) {
- scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey
- if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil {
- return nil, apperr.WrapPostgres(err, "failed to lock message key")
+// LockKeyTx serializes persistence for one idempotency scope.
+func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error {
+ _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope)
+ if err != nil {
+ return apperr.WrapPostgres(err, "failed to lock message key")
}
- hash := messageHash(req)
- var hashVersion int
- var storedHash []byte
- var messageID uuid.UUID
- err := tx.QueryRow(ctx, `SELECT request_hash, message_id, hash_version FROM message_idempotency
- WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`,
- req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&storedHash, &messageID, &hashVersion)
- if err == nil {
- if hashVersion == 1 && len(req.AttachmentIDs) == 0 {
- hash = sha256.Sum256([]byte(req.Text))
- } else if hashVersion != 2 {
- return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
- }
- if !bytes.Equal(hash[:], storedHash) {
- return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
- }
- return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, messageID))
+ return nil
+}
+func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) {
+ v := new(dto.MessageReplay)
+ err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return nil, nil
}
- if !errors.Is(err, pgx.ErrNoRows) {
+ if err != nil {
return nil, apperr.WrapPostgres(err, "failed to read message key")
}
- message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: time.Now().UTC().Truncate(time.Microsecond)}
- if _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`,
- message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt); err != nil {
+ return v, nil
+}
+func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) {
+ return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id))
+}
+func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error {
+ _, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt)
+ if err != nil {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation {
- return nil, apperr.Wrap(err, apperr.CodeNotFound, "chat not found")
+ return apperr.Wrap(err, apperr.CodeNotFound, "chat not found")
}
- return nil, apperr.WrapPostgres(err, "failed to create message")
+ return apperr.WrapPostgres(err, "failed to create message")
}
- for position, attachmentID := range req.AttachmentIDs {
- if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, attachmentID, position); err != nil {
- return nil, apperr.WrapPostgres(err, "failed to link attachment")
+ for position, id := range message.AttachmentIDs {
+ if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil {
+ return apperr.WrapPostgres(err, "failed to link attachment")
}
}
-
- if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version)
- VALUES($1,'POST',$2,$3,$4,$5,clock_timestamp()+interval '24 hours',2)
- ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,
- message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash[:], message.ID); err != nil {
- return nil, apperr.WrapPostgres(err, "failed to save message key")
+ return nil
+}
+func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error {
+ _, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires)
+ if err != nil {
+ return apperr.WrapPostgres(err, "failed to save message key")
}
- return &message, nil
+ return nil
}
func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) {
@@ -120,20 +112,10 @@ func scanMessage(row pgx.Row) (*dto.Message, error) {
const messageColumns = `id, chat_id, sender_id, text, created_at,
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)`
-func messageHash(req dto.CreateMessage) [32]byte {
- // Field order and a non-nil list define the canonical v2 representation.
- canonical := struct {
- Text string `json:"text"`
- AttachmentIDs []uuid.UUID `json:"attachment_ids"`
- }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)}
- encoded, _ := json.Marshal(canonical) // strings and UUIDs have no encoding failures.
- return sha256.Sum256(encoded)
-}
-
func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) {
// Prevent a concurrent integrity update from invalidating the checked rows
// until the message and its links have committed together.
- rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids)
+ rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids)
if err != nil {
return nil, apperr.WrapPostgres(err, "failed to read attachments")
}
@@ -141,7 +123,7 @@ func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, i
result := make([]dto.MessageAttachment, 0, len(ids))
for rows.Next() {
var a dto.MessageAttachment
- if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing); err != nil {
+ if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil {
return nil, apperr.WrapPostgres(err, "failed to read attachment")
}
result = append(result, a)
diff --git a/service/internal/adapter/repository/storage/outbox.go b/service/internal/adapter/repository/storage/outbox.go
new file mode 100644
index 0000000..ffc3191
--- /dev/null
+++ b/service/internal/adapter/repository/storage/outbox.go
@@ -0,0 +1,52 @@
+package storage
+
+import (
+ "context"
+ "errors"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "time"
+)
+
+type OutboxRepo struct{ Pool *pgxpool.Pool }
+
+func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) {
+ _, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt)
+ return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e
+}
+func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) {
+ defer rows.Close()
+ for rows.Next() {
+ var m outbox.AcquiredMessage
+ if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil {
+ return
+ }
+ out = append(out, m)
+ }
+ err = rows.Err()
+ return
+}
+func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) {
+ rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED)
+ UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String())
+ if e != nil {
+ return nil, e
+ }
+ return scanOutbox(rows)
+}
+func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) {
+ rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until 64 {
+ return nil, fmt.Errorf("invalid %s", k)
+ }
+ *p = n
+ }
+ }
+ for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} {
+ if v := os.Getenv(k); v != "" {
+ d, e := time.ParseDuration(v)
+ if e != nil || d <= 0 {
+ return nil, fmt.Errorf("invalid %s", k)
+ }
+ *p = d
+ }
+ }
+ if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency {
+ return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency")
+ }
+ if p := os.Getenv("BROKER_URL_FILE"); p != "" {
+ b, e := os.ReadFile(p)
+ if e != nil {
+ return nil, fmt.Errorf("read broker secret: %w", e)
+ }
+ c.BrokerURL = strings.TrimSpace(string(b))
+ }
+ return c, nil
+}
diff --git a/service/internal/dto/attachment.go b/service/internal/dto/attachment.go
index 76d3a17..1469e14 100644
--- a/service/internal/dto/attachment.go
+++ b/service/internal/dto/attachment.go
@@ -1 +1,48 @@
package dto
+
+import (
+ "gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
+ "github.com/google/uuid"
+ "io"
+ "time"
+)
+
+type AttachmentRecord struct {
+ Public contracts.Attachment
+ Key string
+ StagingKey string
+ PipelineVersion int
+ VariantKeys map[string]string
+ Missing bool
+}
+type UploadAttachment struct {
+ UserID, ChatID uuid.UUID
+ Filename, ContentType string
+ Body io.Reader
+}
+type AttachmentContent struct {
+ Body io.ReadCloser
+ ContentType string
+ Size int64
+}
+type JobMessage struct {
+ Traceparent string `json:"traceparent,omitempty"`
+ Tracestate string `json:"tracestate,omitempty"`
+ SchemaVersion int `json:"schema_version"`
+ JobID uuid.UUID `json:"job_id"`
+ Type string `json:"type"`
+ AttachmentID uuid.UUID `json:"attachment_id"`
+ PipelineVersion int `json:"pipeline_version"`
+ Attempt int `json:"attempt"`
+ CreatedAt time.Time `json:"created_at"`
+}
+type AttachmentJob struct {
+ Message JobMessage
+ State string
+ Token uuid.UUID
+ LeaseUntil *time.Time
+ NextAttemptAt time.Time
+ AttemptLimit int
+ ErrorCode *string
+ UpdatedAt time.Time
+}
diff --git a/service/internal/dto/message.go b/service/internal/dto/message.go
index b85296d..48f33b6 100644
--- a/service/internal/dto/message.go
+++ b/service/internal/dto/message.go
@@ -35,4 +35,11 @@ type MessageAttachment struct {
ID uuid.UUID
ChatID uuid.UUID
Missing bool
+ Status string
+}
+
+type MessageReplay struct {
+ Hash []byte
+ HashVersion int
+ MessageID uuid.UUID
}
diff --git a/service/internal/gen/contracts/types.gen.go b/service/internal/gen/contracts/types.gen.go
index 28d12db..977fa8c 100644
--- a/service/internal/gen/contracts/types.gen.go
+++ b/service/internal/gen/contracts/types.gen.go
@@ -22,7 +22,10 @@ const (
// Defines values for AttachmentStatus.
const (
- Ready AttachmentStatus = "ready"
+ Failed AttachmentStatus = "failed"
+ Processing AttachmentStatus = "processing"
+ Queued AttachmentStatus = "queued"
+ Ready AttachmentStatus = "ready"
)
// Defines values for ErrorErrorCode.
@@ -52,14 +55,17 @@ const (
// Defines values for VariantName.
const (
- Crop VariantName = "crop"
- Optimized VariantName = "optimized"
- Thumbnail VariantName = "thumbnail"
+ VariantNameCrop VariantName = "crop"
+ VariantNameOptimized VariantName = "optimized"
+ VariantNameThumbnail VariantName = "thumbnail"
)
// Defines values for DownloadAttachmentParamsVariant.
const (
- Original DownloadAttachmentParamsVariant = "original"
+ DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop"
+ DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized"
+ DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original"
+ DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail"
)
// Attachment defines model for Attachment.
diff --git a/service/internal/pkg/fault/fault.go b/service/internal/pkg/fault/fault.go
new file mode 100644
index 0000000..0d5af15
--- /dev/null
+++ b/service/internal/pkg/fault/fault.go
@@ -0,0 +1,64 @@
+// Package fault implements explicit, file-controlled barriers for the isolated lab.
+package fault
+
+import (
+ "context"
+ "errors"
+ "github.com/google/uuid"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+func Hit(ctx context.Context, point string, id uuid.UUID) {
+ if os.Getenv("LAB5_FAULTS") != "1" {
+ return
+ }
+ p := filepath.Join("/faults", point+"."+id.String())
+ b, e := os.ReadFile(p)
+ if e != nil {
+ p = filepath.Join("/faults", point+".all")
+ b, e = os.ReadFile(p)
+ if e != nil {
+ return
+ }
+ }
+ hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+ if e != nil {
+ return
+ }
+ _, _ = hit.WriteString(id.String() + "\n")
+ _ = hit.Close()
+ switch strings.TrimSpace(string(b)) {
+ case "crash":
+ _ = os.Remove(p)
+ os.Exit(86)
+ case "pause":
+ for {
+ if _, e = os.Stat(p); os.IsNotExist(e) {
+ return
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(50 * time.Millisecond):
+ }
+ }
+ }
+}
+
+func Error(point string, id uuid.UUID) error {
+ if os.Getenv("LAB5_FAULTS") != "1" {
+ return nil
+ }
+ b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String()))
+ if e != nil {
+ return nil
+ }
+ v := strings.TrimSpace(string(b))
+ if v == "temporary" || v == "permanent" {
+ return errors.New(v)
+ }
+ return nil
+}
diff --git a/service/internal/pkg/imageprocessor/image.go b/service/internal/pkg/imageprocessor/image.go
new file mode 100644
index 0000000..b87678c
--- /dev/null
+++ b/service/internal/pkg/imageprocessor/image.go
@@ -0,0 +1,83 @@
+package imageprocessor
+
+import (
+ "context"
+ "fmt"
+ "image"
+ _ "image/jpeg"
+ _ "image/png"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "time"
+)
+
+func run(ctx context.Context, args ...string) error {
+ cmd := exec.CommandContext(ctx, "magick", args...)
+ cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60")
+ cmd.WaitDelay = time.Second
+ if err := cmd.Run(); err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ return fmt.Errorf("image conversion failed: %w", err)
+ }
+ return nil
+}
+func Validate(ctx context.Context, path string) error {
+ return run(ctx, "-regard-warnings", path, "null:")
+}
+
+type Result struct {
+ Name, Path, MIME string
+ Width, Height int
+}
+
+func fit(w, h, limit int) (int, int) {
+ if w <= limit && h <= limit {
+ return w, h
+ }
+ if w >= h {
+ return limit, max(1, h*limit/w)
+ }
+ return max(1, w*limit/h), limit
+}
+func Process(ctx context.Context, source, dir string, version int) ([]Result, error) {
+ normalized := filepath.Join(dir, "normalized.png")
+ if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil {
+ return nil, e
+ }
+ f, e := os.Open(normalized)
+ if e != nil {
+ return nil, e
+ }
+ cfg, _, e := image.DecodeConfig(f)
+ f.Close()
+ if e != nil {
+ return nil, e
+ }
+ tw, th := fit(cfg.Width, cfg.Height, 256)
+ ow, oh := fit(cfg.Width, cfg.Height, 1280)
+ out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}}
+ for _, v := range out {
+ args := []string{normalized}
+ if v.Name == "crop" {
+ side := min(cfg.Width, cfg.Height)
+ args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage")
+ }
+ args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality")
+ quality := "85"
+ if v.Name == "optimized" && version == 1 {
+ quality = "80"
+ }
+ coder := "jpeg:"
+ if v.Name == "optimized" {
+ coder = "webp:"
+ }
+ args = append(args, quality, coder+v.Path)
+ if e = run(ctx, args...); e != nil {
+ return nil, e
+ }
+ }
+ return out, nil
+}
diff --git a/service/internal/pkg/objectstore/s3.go b/service/internal/pkg/objectstore/s3.go
index 5f90f28..78822da 100644
--- a/service/internal/pkg/objectstore/s3.go
+++ b/service/internal/pkg/objectstore/s3.go
@@ -67,3 +67,11 @@ func Missing(err error) bool {
var api smithy.APIError
return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404")
}
+
+func (s *Store) List(ctx context.Context, token string) (*s3.ListObjectsV2Output, error) {
+ q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)}
+ if token != "" {
+ q.ContinuationToken = &token
+ }
+ return s.Client.ListObjectsV2(ctx, q)
+}
diff --git a/service/internal/service/attachment/interface.go b/service/internal/service/attachment/interface.go
new file mode 100644
index 0000000..71599bf
--- /dev/null
+++ b/service/internal/service/attachment/interface.go
@@ -0,0 +1,51 @@
+package attachment
+
+import (
+ "context"
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "io"
+ "time"
+)
+
+type TransactionManager interface {
+ RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error
+}
+type Outbox interface {
+ CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error)
+}
+type ObjectStore interface {
+ Put(context.Context, string, string, io.Reader, int64) error
+ Get(context.Context, string) (*s3.GetObjectOutput, error)
+ Head(context.Context, string) (*s3.HeadObjectOutput, error)
+ Delete(context.Context, string) error
+ Ping(context.Context) error
+ List(context.Context, string) (*s3.ListObjectsV2Output, error)
+}
+type Repository interface {
+ SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error
+
+ Ping(context.Context) error
+ Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error)
+ GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error)
+ InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error
+ UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error
+ JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error)
+ InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error
+ SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error
+ ExpiredJobs(context.Context) ([]uuid.UUID, error)
+ PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error)
+ FailedOutbox(context.Context) ([]uuid.UUID, error)
+ ListJobs(context.Context) ([]dto.AttachmentJob, error)
+ RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error)
+ AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error
+ Lock(context.Context, string) (*pgxpool.Conn, func(), error)
+ Exists(context.Context, *pgxpool.Conn, string) (bool, error)
+ Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error)
+ MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error)
+ CheckConn(context.Context, *pgxpool.Conn) error
+}
diff --git a/service/internal/service/attachment/pipeline.go b/service/internal/service/attachment/pipeline.go
new file mode 100644
index 0000000..fcdb6cc
--- /dev/null
+++ b/service/internal/service/attachment/pipeline.go
@@ -0,0 +1,486 @@
+package attachment
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
+ "gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/fault"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
+ "gitea.privcloud.tech/mtusi2026/internal/service/outbox"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+var ErrBusy = errors.New("job lease held")
+
+func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) }
+func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) }
+func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
+ now := time.Now().UTC()
+ j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now}
+ if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil {
+ return e
+ }
+ return s.enqueue(ctx, tx, j)
+}
+func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
+ b, e := json.Marshal(j.Message)
+ if e != nil {
+ return e
+ }
+ _, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts})
+ return e
+}
+func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) {
+ a, e := s.Get(ctx, user, id)
+ if e != nil {
+ return nil, e
+ }
+ if a.Public.Status != "ready" {
+ return nil, apperr.New(apperr.CodeConflict, "attachment not ready")
+ }
+ key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes)
+ if variant != "original" {
+ key = a.VariantKeys[variant]
+ if key == "" {
+ return nil, apperr.New(apperr.CodeNotFound, "variant not found")
+ }
+ for _, v := range a.Public.Variants {
+ if string(v.Name) == variant {
+ mime = string(v.ContentType)
+ size = int64(v.SizeBytes)
+ }
+ }
+ }
+ obj, e := s.Store.Get(ctx, key)
+ if e != nil {
+ return nil, Unavailable()
+ }
+ return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil
+}
+func (s *Service) Process(ctx context.Context, m dto.JobMessage) error {
+ var j *dto.AttachmentJob
+ var a *dto.AttachmentRecord
+ claimed := false
+ err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ var e error
+ j, e = s.Repo.JobTx(ctx, tx, m.JobID)
+ if errors.Is(e, pgx.ErrNoRows) {
+ return nil
+ }
+ if e != nil {
+ return e
+ }
+ // Broker data is a notification. Trusted metadata and attempt live in PostgreSQL.
+ if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt {
+ return nil
+ }
+ a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID)
+ if e != nil {
+ return e
+ }
+ if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" {
+ return nil
+ }
+ if j.State == "processing" {
+ return ErrBusy
+ }
+ if time.Now().Before(j.NextAttemptAt) {
+ return ErrBusy
+ }
+ j.State = "processing"
+ j.Token = uuid.New()
+ lease := time.Now().Add(s.Jobs.Lease)
+ j.LeaseUntil = &lease
+ j.UpdatedAt = time.Now().UTC()
+ a.Public.Status = "processing"
+ a.Public.UpdatedAt = j.UpdatedAt
+ if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
+ return e
+ }
+ if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
+ return e
+ }
+ claimed = true
+ return nil
+ })
+ if err != nil || !claimed {
+ return err
+ }
+ s.Fault(ctx, "after_claim", m.JobID)
+ task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout)
+ defer cancel()
+ result, e := s.effects(task, j, a)
+ if e != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ return s.finish(ctx, j, nil, e)
+ }
+ s.Fault(ctx, "after_objects", m.JobID)
+ if err = s.finish(ctx, j, result, nil); err != nil {
+ return err
+ }
+ s.Fault(ctx, "after_result_commit", m.JobID)
+ return nil
+}
+
+type processed struct {
+ Key string
+ Variants []contracts.Variant
+ Keys map[string]string
+}
+type permanentError struct{ code string }
+
+func (e permanentError) Error() string { return e.code }
+func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) {
+ dir, e := os.MkdirTemp("", "pipeline-*")
+ if e != nil {
+ return nil, e
+ }
+ defer os.RemoveAll(dir)
+ key := a.StagingKey
+ if key == "" {
+ key = a.Key
+ }
+ obj, e := s.Store.Get(ctx, key)
+ if e != nil {
+ if objectstore.Missing(e) {
+ return nil, permanentError{"source_missing"}
+ }
+ return nil, e
+ }
+ defer obj.Body.Close()
+ path := filepath.Join(dir, "source")
+ f, e := os.Create(path)
+ if e != nil {
+ return nil, e
+ }
+ hash := sha256.New()
+ n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1))
+ f.Close()
+ if e != nil {
+ return nil, e
+ }
+ if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 {
+ return nil, permanentError{"source_corrupt"}
+ }
+ f, e = os.Open(path)
+ if e != nil {
+ return nil, e
+ }
+ e = s.Validate(ctx, f, string(a.Public.ContentType))
+ f.Close()
+ if e != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ return nil, permanentError{"invalid_image"}
+ }
+ s.Fault(ctx, "before_objects", j.Message.JobID)
+ if e := fault.Error("before_objects", j.Message.JobID); e != nil {
+ if e.Error() == "permanent" {
+ return nil, permanentError{"invalid_image"}
+ }
+ return nil, e
+ }
+ converted, e := imageprocessor.Process(ctx, path, dir, j.Message.PipelineVersion)
+ if e != nil {
+ return nil, e
+ }
+ prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion)
+ result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}}
+ f, e = os.Open(path)
+ if e != nil {
+ return nil, e
+ }
+ e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n)
+ f.Close()
+ if e != nil {
+ return nil, e
+ }
+ s.Fault(ctx, "after_original_put", j.Message.JobID)
+ for _, v := range converted {
+ b, e := os.ReadFile(v.Path)
+ if e != nil {
+ return nil, e
+ }
+ sum := fmt.Sprintf("%x", sha256.Sum256(b))
+ key := prefix + v.Name + "/" + sum
+ if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil {
+ return nil, e
+ }
+ result.Keys[v.Name] = key
+ result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum})
+ }
+ return result, nil
+}
+func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error {
+ return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID)
+ if e != nil {
+ return e
+ }
+ a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
+ if e != nil {
+ return e
+ }
+ if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion {
+ return nil
+ }
+ if cause != nil {
+ return s.retry(ctx, tx, j, a, cause)
+ }
+
+ if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil {
+ return e
+ }
+ for _, v := range result.Variants {
+ if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil {
+ return e
+ }
+ }
+ j.State = "ready"
+ j.ErrorCode = nil
+ j.LeaseUntil = nil
+ j.UpdatedAt = time.Now().UTC()
+ a.Missing = false
+ a.Key = result.Key
+ a.VariantKeys = result.Keys
+ a.Public.Variants = result.Variants
+ a.Public.Status = "ready"
+ a.Public.ErrorCode = nil
+ a.Public.UpdatedAt = j.UpdatedAt
+ if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
+ return e
+ }
+ return s.Repo.UpdateTx(ctx, tx, a)
+ })
+}
+func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error {
+ code := "processing_failed"
+ var permanent permanentError
+ terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit
+ if permanent.code != "" {
+ code = permanent.code
+ }
+ if terminal && permanent.code == "" {
+ code = "retry_exhausted"
+ }
+ j.LeaseUntil = nil
+ j.Token = uuid.New()
+ j.UpdatedAt = time.Now().UTC()
+ j.ErrorCode = &code
+ if terminal {
+ j.State = "failed"
+ a.Public.Status = "failed"
+ a.Public.ErrorCode = &code
+ } else {
+ j.State = "queued"
+ j.Message.Attempt++
+ j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1))
+ a.Public.Status = "queued"
+ a.Public.ErrorCode = nil
+ if e := s.enqueue(ctx, tx, j); e != nil {
+ return e
+ }
+ }
+ a.Public.UpdatedAt = j.UpdatedAt
+ if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil {
+ return e
+ }
+ return s.Repo.UpdateTx(ctx, tx, a)
+}
+func (s *Service) Recover(ctx context.Context) error {
+ ids, e := s.Repo.ExpiredJobs(ctx)
+ if e != nil {
+ return e
+ }
+ for _, id := range ids {
+ e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ j, e := s.Repo.JobTx(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) {
+ return nil
+ }
+ a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
+ if e != nil {
+ return e
+ }
+ if a.PipelineVersion != j.Message.PipelineVersion {
+ return nil
+ }
+ return s.retry(ctx, tx, j, a, errors.New("lease expired"))
+ })
+ if e != nil {
+ return e
+ }
+ }
+ dead, e := s.Repo.FailedOutbox(ctx)
+ if e != nil {
+ return e
+ }
+ for _, id := range dead {
+ if e = s.PublishFailed(ctx, id); e != nil {
+ return e
+ }
+ }
+ return nil
+}
+func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error {
+ return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ j, e := s.Repo.JobTx(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ if j.State != "queued" {
+ return nil
+ }
+ dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt)
+ if e != nil {
+ return e
+ }
+ if !dead {
+ return nil
+ }
+ a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
+ if e != nil {
+ return e
+ }
+ if a.PipelineVersion != j.Message.PipelineVersion {
+ return nil
+ }
+ code := "publish_failed"
+ j.State = "failed"
+ j.ErrorCode = &code
+ j.UpdatedAt = time.Now().UTC()
+ a.Public.Status = "failed"
+ a.Public.ErrorCode = &code
+ a.Public.UpdatedAt = j.UpdatedAt
+ if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
+ return e
+ }
+ return s.Repo.UpdateTx(ctx, tx, a)
+ })
+}
+func (s *Service) Replay(ctx context.Context, id uuid.UUID) error {
+ return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ j, e := s.Repo.JobTx(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
+ if e != nil {
+ return e
+ }
+ if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion {
+ return apperr.New(apperr.CodeConflict, "only current failed job can be replayed")
+ }
+ source := a.StagingKey
+ if source == "" {
+ return errors.New("source unavailable; restore original first")
+ }
+ if _, e = s.Store.Head(ctx, source); e != nil {
+ return errors.New("source unavailable")
+ }
+ j.State = "queued"
+ j.Message.Attempt++
+ j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1
+ j.NextAttemptAt = time.Now().UTC()
+ j.UpdatedAt = j.NextAttemptAt
+ j.Token = uuid.New()
+ j.ErrorCode = nil
+ j.LeaseUntil = nil
+ a.Public.Status = "queued"
+ a.Public.ErrorCode = nil
+ a.Public.UpdatedAt = j.UpdatedAt
+ if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
+ return e
+ }
+ if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
+ return e
+ }
+ if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil {
+ return e
+ }
+ return s.enqueue(ctx, tx, j)
+ })
+}
+func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error {
+ if version < 1 || version > 2 {
+ return errors.New("supported pipeline versions: 1, 2")
+ }
+ return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ a, e := s.Repo.GetTx(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ if version <= a.PipelineVersion || a.Public.Status != "ready" {
+ return errors.New("requires ready attachment and newer version")
+ }
+ a.PipelineVersion = version
+ a.StagingKey = a.Key
+ a.Public.Status = "queued"
+ a.Public.Variants = []contracts.Variant{}
+ a.VariantKeys = map[string]string{}
+ a.Public.UpdatedAt = time.Now().UTC()
+ if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
+ return e
+ }
+ return s.newJob(ctx, tx, a)
+ })
+}
+func (s *Service) Cleanup(ctx context.Context) error {
+ records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention))
+ if e != nil {
+ return e
+ }
+ for _, record := range records {
+ // Serialize deletion with operator Restore and the initial upload handoff.
+ _, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey)
+ if lockErr != nil {
+ return lockErr
+ }
+ e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ a, e := s.Repo.GetTx(ctx, tx, record.Public.Id)
+ if e != nil {
+ return e
+ }
+ if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" {
+ return nil
+ }
+ if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) {
+ return nil
+ }
+ if !strings.HasPrefix(a.StagingKey, "results/") {
+ if e = s.Store.Delete(ctx, a.StagingKey); e != nil {
+ return e
+ }
+ }
+ a.StagingKey = ""
+ return s.Repo.UpdateTx(ctx, tx, a)
+ })
+ unlock()
+ if e != nil {
+ return e
+ }
+ }
+ return nil
+}
+func managedPipelineKey(key string) bool {
+ return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/")
+}
diff --git a/service/internal/service/attachment/reconcile.go b/service/internal/service/attachment/reconcile.go
index 0a11258..0ad54ad 100644
--- a/service/internal/service/attachment/reconcile.go
+++ b/service/internal/service/attachment/reconcile.go
@@ -10,8 +10,9 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
- "github.com/aws/aws-sdk-go-v2/service/s3"
+
"github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
)
@@ -38,17 +39,17 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
result.Apply = apply
result.GraceSeconds = s.Config.Grace.Seconds()
defer func() { result.DurationMS = time.Since(start).Milliseconds() }()
- if err = s.Repo.Pool.Ping(ctx); err != nil {
+ if err = s.Repo.Ping(ctx); err != nil {
return
}
result.S3Requests++
if err = s.Store.Ping(ctx); err != nil {
return
}
- pager := s3.NewListObjectsV2Paginator(s.Store.Client, &s3.ListObjectsV2Input{Bucket: &s.Store.Bucket, Prefix: aws.String(Prefix), MaxKeys: aws.Int32(100)})
- for pager.HasMorePages() {
+ token := ""
+ for {
result.S3Requests++
- page, e := pager.NextPage(ctx)
+ page, e := s.Store.List(ctx, token)
if e != nil {
return result, e
}
@@ -92,7 +93,7 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
return nil
}
result.DBChecks++
- if _, e = conn.Exec(ctx, "SELECT 1"); e != nil {
+ if e = s.Repo.CheckConn(ctx, conn); e != nil {
return e
}
result.S3Requests++
@@ -106,7 +107,12 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
return result, e
}
}
+ if !aws.ToBool(page.IsTruncated) {
+ break
+ }
+ token = aws.ToString(page.NextContinuationToken)
}
+
after := uuid.Nil
for {
result.DBChecks++
@@ -120,6 +126,9 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
for _, a := range page {
result.Metadata++
after = a.Public.Id
+ if a.Public.Status != "ready" {
+ continue
+ }
result.LockAttempts++
conn, unlock, e := s.Repo.Lock(ctx, a.Key)
if e != nil {
@@ -201,5 +210,19 @@ func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error
return fmt.Errorf("stored original verification failed")
}
_, e = s.Repo.MarkMissing(ctx, conn, id, false)
- return e
+ if e != nil {
+ return e
+ }
+ return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ current, e := s.Repo.GetTx(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ if current.Public.Status == "failed" {
+ current.StagingKey = current.Key
+ current.Public.UpdatedAt = time.Now().UTC()
+ return s.Repo.UpdateTx(ctx, tx, current)
+ }
+ return nil
+ })
}
diff --git a/service/internal/service/attachment/service.go b/service/internal/service/attachment/service.go
index 36c5d71..686bc13 100644
--- a/service/internal/service/attachment/service.go
+++ b/service/internal/service/attachment/service.go
@@ -8,8 +8,7 @@ import (
_ "image/jpeg"
_ "image/png"
"io"
- "mime"
- "net/http"
+
"os"
"strings"
"time"
@@ -18,28 +17,31 @@ import (
"github.com/google/uuid"
- "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
- "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
+ "github.com/jackc/pgx/v5"
)
const Prefix = "originals/"
+const StagingPrefix = "staging/"
type Membership interface {
GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error)
}
type Service struct {
- Repo *storage.AttachmentsRepo
- Store *objectstore.Store
+ Repo Repository
+ Store ObjectStore
Config *config.Attachment
chats Membership
+ tm TransactionManager
+ outbox Outbox
+ Jobs *config.Pipeline
uploads, decoders chan struct{}
}
-func New(repo *storage.AttachmentsRepo, store *objectstore.Store, c *config.Attachment, chats Membership) *Service {
+func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service {
return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)}
}
func ValidFilename(s string) bool {
@@ -53,21 +55,15 @@ func ValidFilename(s string) bool {
}
return true
}
-func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid.UUID, filename string) (*contracts.Attachment, error) {
- ctx := r.Context()
- if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: chat}); e != nil {
+func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) {
+ s.tm = tm
+ s.outbox = outbox
+ s.Jobs = jobs
+}
+func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (*contracts.Attachment, error) {
+ if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil {
return nil, e
}
- if !ValidFilename(filename) {
- return nil, apperr.New(apperr.CodeBadRequest, "invalid filename")
- }
- mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
- if e != nil || (mt != "image/png" && mt != "image/jpeg") {
- return nil, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")
- }
- if r.ContentLength > s.Config.MaxBytes {
- return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
- }
select {
case s.uploads <- struct{}{}:
defer func() { <-s.uploads }()
@@ -80,23 +76,22 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid
}
defer os.Remove(f.Name())
defer f.Close()
- r.Body = http.MaxBytesReader(w, r.Body, s.Config.MaxBytes)
hash := sha256.New()
- size, e := io.Copy(io.MultiWriter(f, hash), r.Body)
+ size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1))
if e != nil {
- if _, ok := e.(*http.MaxBytesError); ok {
- return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
- }
return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload")
}
- if e = s.Validate(ctx, f, mt); e != nil {
+ if size > s.Config.MaxBytes {
+ return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
+ }
+ if e = s.Validate(ctx, f, req.ContentType); e != nil {
return nil, e
}
now := time.Now().UTC().Truncate(time.Microsecond)
id := uuid.New()
- key := Prefix + id.String()
- a := &storage.AttachmentRecord{Key: key, Public: contracts.Attachment{Id: id, ChatId: chat, OwnerId: user, Filename: filename, ContentType: contracts.AttachmentContentType(mt), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "ready", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}}
- conn, unlock, e := s.Repo.Lock(ctx, key)
+ key := StagingPrefix + id.String()
+ a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}}
+ _, unlock, e := s.Repo.Lock(ctx, key)
if e != nil {
return nil, e
}
@@ -104,14 +99,19 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid
if _, e = f.Seek(0, 0); e != nil {
return nil, e
}
- if e = s.Store.Put(ctx, key, mt, f, size); e != nil {
+ if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil {
return nil, Unavailable()
}
- // Never blindly compensate a failed/ambiguous COMMIT with DELETE: a committed
- // row may already reference this key. Reconciliation decides after grace.
- if e = s.Repo.Save(ctx, conn, a); e != nil {
+ s.Fault(ctx, "after_staging", id)
+ if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
+ if e := s.Repo.InsertTx(ctx, tx, a); e != nil {
+ return e
+ }
+ return s.newJob(ctx, tx, a)
+ }); e != nil {
return nil, e
}
+ s.Fault(ctx, "after_upload_commit", id)
return &a.Public, nil
}
func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error {
@@ -134,12 +134,12 @@ func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error {
if _, e = f.Seek(0, 0); e != nil {
return e
}
- if _, _, e = image.Decode(f); e != nil {
+ if e = validateImage(ctx, f.Name()); e != nil {
return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image")
}
return ctx.Err()
}
-func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.AttachmentRecord, error) {
+func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) {
a, e := s.Repo.Get(ctx, id)
if e != nil {
return nil, e
@@ -154,6 +154,9 @@ func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.Attachm
}
func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") }
func ManagedKey(key string) bool {
+ if managedPipelineKey(key) {
+ return true
+ }
if !strings.HasPrefix(key, Prefix) {
return false
}
diff --git a/service/internal/service/chat/idempotency.go b/service/internal/service/chat/idempotency.go
new file mode 100644
index 0000000..13fa962
--- /dev/null
+++ b/service/internal/service/chat/idempotency.go
@@ -0,0 +1,54 @@
+package chat
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "gitea.privcloud.tech/mtusi2026/internal/dto"
+ "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "time"
+)
+
+// CreateIdempotentTx performs the message use case inside an existing transaction.
+func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) {
+ scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey
+ if err := repo.LockKeyTx(ctx, tx, scope); err != nil {
+ return nil, err
+ }
+ canonical := struct {
+ Text string `json:"text"`
+ AttachmentIDs []uuid.UUID `json:"attachment_ids"`
+ }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)}
+ encoded, err := json.Marshal(canonical)
+ if err != nil {
+ return nil, err
+ }
+ hash := sha256.Sum256(encoded)
+ replay, err := repo.ReplayTx(ctx, tx, req)
+ if err != nil {
+ return nil, err
+ }
+ if replay != nil {
+ if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 {
+ hash = sha256.Sum256([]byte(req.Text))
+ } else if replay.HashVersion != 2 {
+ return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
+ }
+ if !bytes.Equal(hash[:], replay.Hash) {
+ return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
+ }
+ return repo.MessageTx(ctx, tx, replay.MessageID)
+ }
+ now := time.Now().UTC().Truncate(time.Microsecond)
+ message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now}
+ if err = repo.InsertMessageTx(ctx, tx, message); err != nil {
+ return nil, err
+ }
+ if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil {
+ return nil, err
+ }
+ return &message, nil
+}
diff --git a/service/internal/service/chat/interface.go b/service/internal/service/chat/interface.go
index dbe7157..17c7742 100644
--- a/service/internal/service/chat/interface.go
+++ b/service/internal/service/chat/interface.go
@@ -2,6 +2,7 @@ package chat
import (
"context"
+ "time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -17,7 +18,11 @@ type chatsRepo interface {
type messagesRepo interface {
AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error)
- CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error)
+ LockKeyTx(context.Context, pgx.Tx, string) error
+ ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error)
+ MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error)
+ InsertMessageTx(context.Context, pgx.Tx, dto.Message) error
+ SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error
GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error)
ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error)
}
diff --git a/service/internal/service/chat/svc.go b/service/internal/service/chat/svc.go
index 6dacf96..700f168 100644
--- a/service/internal/service/chat/svc.go
+++ b/service/internal/service/chat/svc.go
@@ -190,6 +190,9 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M
if attachment.ChatID != req.ChatID {
return apperr.New(apperr.CodeNotFound, "attachment not found")
}
+ if attachment.Status == "failed" {
+ return apperr.New(apperr.CodeConflict, "failed attachment")
+ }
if attachment.Missing {
return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable")
}
@@ -197,7 +200,7 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M
}
var err error
- message, err = c.messagesRepo.CreateIdempotentTx(ctx, tx, req)
+ message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req)
return err
})
return message, err
diff --git a/service/internal/service/outbox/api.go b/service/internal/service/outbox/api.go
new file mode 100644
index 0000000..aaac1a7
--- /dev/null
+++ b/service/internal/service/outbox/api.go
@@ -0,0 +1,12 @@
+package outbox
+
+import (
+ "context"
+
+ "github.com/jackc/pgx/v5"
+)
+
+type API interface {
+ CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error)
+ Create(context.Context, CreateMessageReq) (CreateMessageRes, error)
+}
diff --git a/service/internal/service/outbox/models.go b/service/internal/service/outbox/models.go
new file mode 100644
index 0000000..fc90e0e
--- /dev/null
+++ b/service/internal/service/outbox/models.go
@@ -0,0 +1,115 @@
+package outbox
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+)
+
+type CreateMessageReq struct {
+ ID uuid.UUID
+ CreatedAt time.Time
+ Transport Transport
+ AggregateType string
+ AggregateID uuid.UUID
+ MessageType string
+ IdempotencyKey *string
+ SchemaVersion int
+ CausationID *uuid.UUID
+ Metadata map[string]string
+ Payload []byte
+ MaxAttempts int
+ Status Status
+}
+
+type CreateMessageRes struct {
+ ID uuid.UUID
+ CreatedAt time.Time
+}
+
+type MarkPublishedMessageReq struct {
+ ID uuid.UUID
+ UpdatedAt time.Time
+ PublishedAt time.Time
+}
+
+type MarkFailedMessageReq struct {
+ ID uuid.UUID
+ UpdatedAt time.Time
+ NextAttemptAt time.Time
+ LastError string
+}
+
+type MarkDeadLetterMessageReq struct {
+ ID uuid.UUID
+ UpdatedAt time.Time
+ DeadLetterAt time.Time
+ LastError string
+}
+
+type AcquireMessagesForProcessingReq struct {
+ Limit uint
+ LockDuration time.Duration
+}
+
+type AcquiredMessage struct {
+ Token uuid.UUID
+ ID uuid.UUID
+ Transport Transport
+ AggregateType string
+ AggregateID uuid.UUID
+ MessageType string
+ SchemaVersion int
+ CorrelationID uuid.UUID
+ CausationID *uuid.UUID
+ Status Status
+ Metadata map[string]string
+ Payload []byte
+ Attempts int
+ MaxAttempts int
+ LastError *string
+ LockedAt *time.Time
+ LockedUntil *time.Time
+ NextAttemptAt *time.Time
+ PublishedAt *time.Time
+ DeadLetterAt *time.Time
+ CreatedAt time.Time
+ UpdatedAt *time.Time
+}
+
+type RecoverStuckMessagesAsFailedReq struct {
+ UpdatedAt time.Time
+ NextAttemptAt time.Time
+ LastError string
+ Limit uint
+}
+
+type RecoverStuckMessagesAsDeadLetterReq struct {
+ UpdatedAt time.Time
+ DeadLetterAt time.Time
+ LastError string
+ Limit uint
+}
+
+type DeliveryMessage struct {
+ ID uuid.UUID
+ Transport Transport
+ AggregateType string
+ AggregateID uuid.UUID
+ MessageType string
+ SchemaVersion int
+ CorrelationID uuid.UUID
+ CausationID *uuid.UUID
+ Payload []byte
+ Metadata map[string]string
+}
+
+type ProcessOnceResult struct {
+ RecoveredAsFailed int64
+ RecoveredAsDeadLetter int64
+
+ Acquired int
+ Published int
+ Failed int
+ DeadLettered int
+}
diff --git a/service/internal/service/outbox/status.go b/service/internal/service/outbox/status.go
new file mode 100644
index 0000000..c7a1d3a
--- /dev/null
+++ b/service/internal/service/outbox/status.go
@@ -0,0 +1,16 @@
+package outbox
+
+const (
+ StatusCreated Status = "created"
+ StatusPending Status = "pending"
+ StatusProcessing Status = "processing"
+ StatusPublished Status = "published"
+ StatusFailed Status = "failed"
+ StatusDeadLetter Status = "dead_letter"
+)
+
+type Status string
+
+func (s Status) String() string {
+ return string(s)
+}
diff --git a/service/internal/service/outbox/svc.go b/service/internal/service/outbox/svc.go
new file mode 100644
index 0000000..7dd7faa
--- /dev/null
+++ b/service/internal/service/outbox/svc.go
@@ -0,0 +1,114 @@
+// Adapted from mamont-optics/application/internal/service/outbox.
+// Retains transactional creation, batch acquisition, bounded retries and lease recovery.
+package outbox
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "math/rand/v2"
+ "time"
+)
+
+type Repository interface {
+ CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error)
+ AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error)
+ Expired(context.Context) ([]AcquiredMessage, error)
+ Update(context.Context, AcquiredMessage, Status, time.Time, string) error
+}
+type TransportPublisher interface {
+ Publish(context.Context, DeliveryMessage) error
+}
+type TransactionManager interface {
+ RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error
+}
+type Svc struct {
+ repo Repository
+ tm TransactionManager
+ publisher TransportPublisher
+ OnDead func(context.Context, uuid.UUID) error
+}
+
+func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc {
+ return &Svc{repo: repo, tm: tm, publisher: publisher}
+}
+func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) {
+ if r.ID == uuid.Nil {
+ r.ID = uuid.New()
+ }
+ if r.CreatedAt.IsZero() {
+ r.CreatedAt = time.Now().UTC()
+ }
+ if r.Status == "" {
+ r.Status = StatusPending
+ }
+ if r.MaxAttempts == 0 {
+ r.MaxAttempts = 5
+ }
+ return s.repo.CreateTx(ctx, tx, r)
+}
+func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) {
+ err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e })
+ return
+}
+func Backoff(attempt int) time.Duration {
+ n := min(max(attempt-1, 0), 5)
+ cap := min(time.Second*time.Duration(1<= m.MaxAttempts {
+ status = StatusDeadLetter
+ }
+ if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil {
+ return err
+ }
+ if status == StatusDeadLetter && s.OnDead != nil {
+ var j struct {
+ JobID uuid.UUID `json:"job_id"`
+ }
+ if err := json.Unmarshal(m.Payload, &j); err != nil {
+ return err
+ }
+ return s.OnDead(ctx, j.JobID)
+ }
+ return nil
+}
diff --git a/service/internal/service/outbox/transport.go b/service/internal/service/outbox/transport.go
new file mode 100644
index 0000000..a92608c
--- /dev/null
+++ b/service/internal/service/outbox/transport.go
@@ -0,0 +1,7 @@
+package outbox
+
+type Transport string
+
+const TransportAMQP Transport = "amqp"
+
+func (t Transport) String() string { return string(t) }
diff --git a/service/migrations/008_async_pipeline.sql b/service/migrations/008_async_pipeline.sql
new file mode 100644
index 0000000..4607e20
--- /dev/null
+++ b/service/migrations/008_async_pipeline.sql
@@ -0,0 +1,41 @@
+ALTER TABLE attachments
+ ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')),
+ ADD COLUMN error_code text,
+ ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
+ ADD COLUMN staging_key text NOT NULL DEFAULT '',
+ ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
+ ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}';
+UPDATE attachments SET staging_key=object_key;
+CREATE TABLE attachment_jobs (
+ id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
+ pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0),
+ attempt_limit integer NOT NULL, state text NOT NULL,
+ token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL,
+ error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL,
+ UNIQUE(attachment_id,pipeline_version)
+);
+CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until);
+CREATE TABLE outbox (
+ id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
+ payload jsonb NOT NULL, status text NOT NULL,
+ attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL,
+ token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL,
+ last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL
+);
+CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at);
+CREATE TABLE job_audit (
+ id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
+ action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp()
+);
+-- Existing lab4 originals become durable sources for the first asynchronous run.
+INSERT INTO attachment_jobs
+ SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments;
+INSERT INTO outbox
+ SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at),
+ 'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs;
+
+---- create above / drop below ----
+DROP TABLE job_audit;
+DROP TABLE outbox;
+DROP TABLE attachment_jobs;
+ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys;
diff --git a/service/migrations/009_variant_manifest.sql b/service/migrations/009_variant_manifest.sql
new file mode 100644
index 0000000..84413c8
--- /dev/null
+++ b/service/migrations/009_variant_manifest.sql
@@ -0,0 +1,8 @@
+CREATE TABLE attachment_variants (
+ attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
+ variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')),
+ object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL,
+ PRIMARY KEY(attachment_id,pipeline_version,variant)
+);
+---- create above / drop below ----
+DROP TABLE attachment_variants;
diff --git a/service/tests/attachments_test.go b/service/tests/attachments_test.go
index e9a5860..1a3e6bf 100644
--- a/service/tests/attachments_test.go
+++ b/service/tests/attachments_test.go
@@ -69,11 +69,12 @@ func TestAttachmentContractAndErrors(t *testing.T) {
Sha string `json:"sha256"`
Size int `json:"size_bytes"`
}
- decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 201, &att)
+ decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att)
hash := sha256.Sum256(data)
if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) {
t.Fatal("original metadata mismatch")
}
+ waitAttachmentReady(t, api, alice, att.ID)
content := "/api/v1/attachments/" + att.ID + "/content"
got := api.request(t, "GET", content, bob, "")
if got.status != 200 || !bytes.Equal(got.body, data) {
@@ -115,7 +116,7 @@ func TestAttachmentContractAndErrors(t *testing.T) {
boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk))
boundary = append(boundary, data[len(data)-12:]...)
var boundaryAtt map[string]any
- decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 201, &boundaryAtt)
+ decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt)
oversized := make([]byte, 10485761)
copy(oversized, data)
@@ -127,7 +128,7 @@ func TestAttachmentContractAndErrors(t *testing.T) {
t.Fatal(e)
}
var jpegAtt map[string]any
- decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 201, &jpegAtt)
+ decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt)
imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}`
var message map[string]any
key := uuid.NewString()
@@ -145,7 +146,9 @@ func TestAttachmentContractAndErrors(t *testing.T) {
for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} {
assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request")
}
- assertError(t, api.request(t, "GET", content+"?variant=thumbnail", bob, ""), 400, "invalid_request")
+ if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 {
+ t.Fatal("thumbnail unavailable")
+ }
}
func TestConcurrentAttachmentMessage(t *testing.T) {
api := newAPI(t)
@@ -154,7 +157,7 @@ func TestConcurrentAttachmentMessage(t *testing.T) {
var att struct {
ID string `json:"id"`
}
- decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 201, &att)
+ decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att)
key := uuid.NewString()
body := `{"text":"","attachment_ids":["` + att.ID + `"]}`
type result struct {
@@ -202,3 +205,22 @@ func TestConcurrentAttachmentMessage(t *testing.T) {
t.Fatal("duplicate history")
}
}
+
+func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) {
+ t.Helper()
+ deadline := time.Now().Add(60 * time.Second)
+ for time.Now().Before(deadline) {
+ var att struct {
+ Status string `json:"status"`
+ }
+ decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att)
+ if att.Status == "ready" {
+ return
+ }
+ if att.Status == "failed" {
+ t.Fatal("attachment failed")
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ t.Fatal("attachment readiness deadline")
+}
diff --git a/student-tests/README.md b/student-tests/README.md
new file mode 100644
index 0000000..6e922fb
--- /dev/null
+++ b/student-tests/README.md
@@ -0,0 +1,14 @@
+# Проверки решения lab5
+
+Публичные `tests/` не изменены. Собственные проверки используют их HTTP-клиент и создают только синтетические данные в локальном стенде.
+
+- `make test-images`: реальные форматы и размеры вариантов, floor/no-upscale, EXIF orientation, белый фон alpha, центральный crop, hashes, ACL и отрицательные входы.
+- `make test-staging`: защита queued-source, удаление staging после ready, terminal source_missing, 24h retention и восстановление с явным replay. Для retention меняется только возраст одной синтетической записи на 25h; реальный день не ожидается и конфигурация retention не сокращается.
+- `make test-broker-persistence`: заменяет контейнер брокера при остановленных worker и dispatcher; проверяет, что уже опубликованное сообщение осталось в очереди с тем же volume.
+- `make test-pipeline`: 13 сценариев × 3 повтора, включая crash, дубли, lease recovery, reorder, устаревший worker и версию pipeline. На время выполнения подключает `compose.faults.yaml`, перезапускает процессы и stores с сохранением volumes. В конце восстанавливает обычный Compose без fault injection.
+
+Перед запуском нужен `make up`. Проверки, которые останавливают/перезапускают процессы (`test-staging`, `test-pipeline`, persistence/failover), запускаются последовательно. Результаты — `evidence/*.json`; секреты и токены туда не записываются. Директивы fault injection лежат в игнорируемой `.lab5-faults/`.
+
+`jobs redeliver --id JOB_UUID` — только инструмент экспериментального профиля с `LAB5_FAULTS=1`, публикующий дополнительную копию сохранённой job. Обычный оператор использует `jobs replay` для failed или `jobs reprocess` для новой версии.
+
+Исследование и ограничения описаны в [REPORT.md](../REPORT.md).
diff --git a/student-tests/broker_persistence.py b/student-tests/broker_persistence.py
new file mode 100644
index 0000000..6d63bc9
--- /dev/null
+++ b/student-tests/broker_persistence.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Prove a confirmed queued message survives broker container replacement, not only restart."""
+from pathlib import Path
+import json
+import subprocess
+import sys
+import time
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tests'))
+from client import Client,check
+
+def compose(*args):
+ r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150)
+ check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip()
+
+def queued():
+ r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30)
+ if r.returncode:return 0
+ out=r.stdout
+ for line in out.splitlines():
+ fields=line.split()
+ if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1])
+ return 0
+
+def main():
+ c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap()
+ try:
+ compose('stop','worker');att,data=c.attachment(alice,chat)
+ deadline=time.monotonic()+20
+ while queued()<1 and time.monotonic()=1,'message not durably queued before replacement')
+ published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id'])
+ check(published=='t','outbox not marked published before replacement')
+ compose('stop','dispatcher')
+ old=compose('ps','-q','broker')
+ compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker')
+ new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced')
+ deadline=time.monotonic()+30
+ while queued()<1 and time.monotonic()=1,'confirmed queue message lost after replacement')
+ compose('start','worker');c.wait_ready(bob,att)
+ check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement')
+ result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True}
+ (ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n')
+ print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity')
+ finally:compose('start','worker','dispatcher')
+if __name__=='__main__':main()
diff --git a/student-tests/images.py b/student-tests/images.py
new file mode 100644
index 0000000..4b32463
--- /dev/null
+++ b/student-tests/images.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Image contract checks using synthetic fixtures and real decoding in ImageMagick."""
+import hashlib
+import json
+from pathlib import Path
+import struct
+import subprocess
+import sys
+import urllib.parse
+import zlib
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tests'))
+from client import Client,check
+from image_probe import jpeg_size
+
+def magick(args,data=None):
+ r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30)
+ check(r.returncode==0,'fixture decoder failed')
+ return r.stdout
+
+def png(w,h,rgba):
+ def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b))
+ return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'')
+
+def main():
+ c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[]
+ jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255)))
+ # EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise.
+ tiff=b'II'+struct.pack('H',len(exif)+2)+exif+jpeg[2:]
+ for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]:
+ att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att)
+ original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed')
+ for v in ready['variants']:
+ b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch')
+ actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split()))
+ check(actual==(v['width'],v['height']),'actual dimensions differ')
+ if v['name']=='thumbnail':
+ check(jpeg_size(b)==dimensions,'thumbnail geometry differs')
+ if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white')
+ if v['name']=='crop':
+ check(actual==(128,128),'crop geometry differs')
+ if name=='center_crop':
+ import re
+ color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b))))
+ check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered')
+ check(b'Exif\0\0' not in b,'EXIF retained in derivative')
+ c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve)
+ results.append({'case':name,'result':'PASS','thumbnail':dimensions})
+ for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]:
+ c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime})
+ (ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs')
+if __name__=='__main__':main()
diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py
new file mode 100644
index 0000000..6e88ea6
--- /dev/null
+++ b/student-tests/pipeline.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens."""
+import argparse
+import concurrent.futures
+import datetime
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+import uuid
+
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tests'))
+from client import Client, Failure, check
+from image_probe import jpeg_size
+FAULTS=ROOT/'.lab5-faults'
+COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')]
+RESULTS=[]
+SELECTED=set()
+
+def compose(*args, allow_failure=False):
+ p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180)
+ if p.returncode and not allow_failure:
+ # Infrastructure logs may contain settings; report only action and exit status.
+ raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode))
+ return p
+
+def sql(query):
+ return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip()
+
+def uid(value):
+ return str(uuid.UUID(value))
+
+def job(att,version=1):
+ return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version))
+
+def operator(action,id=None,version=None, allow_failure=False):
+ args=['run','--rm','--no-deps','-T','operator','jobs',action]
+ if id: args+=['--id',uid(id)]
+ if version:args+=['--pipeline-version',str(version)]
+ return compose(*args,allow_failure=allow_failure)
+
+def wait(predicate,timeout=65):
+ deadline=time.monotonic()+timeout
+ while time.monotonic()=2,'lease not recovered')
+ operator('reprocess',att['id'],2);second=ready(c,bob,att,data)
+ check(first['variants']!=second['variants'],'pipeline versions are indistinguishable')
+ operator('redeliver',jid);marker.unlink();time.sleep(1)
+ check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2')
+ check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back')
+ ready(c,bob,att,data)
+ return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8}
+ record('lease_reorder_and_stale_version',run,version_fence)
+ def retries():
+ att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65)
+ check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored')
+ c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
+ marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed')
+ return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6}
+ record('bounded_retry_and_explicit_replay',run,retries)
+ def permanent():
+ att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid}
+ record('permanent_error',run,permanent)
+ for point in ('after_staging','after_upload_commit'):
+ def api_crash(point=point):
+ if point=='after_upload_commit':compose('stop','dispatcher')
+ marker=directive(point,'all','crash')
+ try:c.attachment(alice,chat)
+ except Failure:pass
+ aid=hit(marker);att={'id':aid}
+ if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists')
+ else:
+ check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached')
+ compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes())
+ wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'}
+ record('api_crash_'+point,run,api_crash)
+ payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS}
+ check(bool(RESULTS),'no scenarios selected')
+ payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab5','--format','{{.Id}}'],text=True).strip()
+ (ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n')
+ finally:
+ for f in FAULTS.iterdir():f.unlink()
+ compose('stop','worker','dispatcher',allow_failure=True)
+ # Restore standard configuration, disabling fault injection and short experiment leases.
+ subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180)
+
+def health(c):
+ try:c.call('GET','/health/ready',schema='Health');return True
+ except Failure:return False
+
+if __name__=='__main__':main()
diff --git a/student-tests/storage.py b/student-tests/storage.py
new file mode 100644
index 0000000..e652d8d
--- /dev/null
+++ b/student-tests/storage.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Staging protection, retirement, retention and explicit restoration on synthetic data."""
+import json
+from pathlib import Path
+import sys
+import time
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tests'))
+from client import Client,check
+BASE=[str(ROOT/'scripts/compose.sh')]
+import subprocess
+
+def command(*args,fail=False):
+ r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120)
+ if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0])
+ return r
+
+def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip()
+def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail)
+def wait(fn):
+ deadline=time.monotonic()+20
+ while time.monotonic()0 and exists(key),'reconciliation removed active source')
+ results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']})
+ command('start','worker');c.wait_ready(bob,att)
+ wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='')
+ check(not exists(key),'ready source staging not retired')
+ results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']})
+ command('stop','worker')
+ failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id'])
+ op('storage','remove-object','--id',failed['id']);command('start','worker')
+ def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed'
+ wait(is_failed)
+ check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent')
+ check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source')
+ op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png')
+ check(exists(key),'restored source missing')
+ # Advance only the synthetic record's age; do not wait a real day or change production retention.
+ sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id'])
+ wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='')
+ check(not exists(key),'expired failed source not retired')
+ check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source')
+ results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25})
+ op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed)
+ check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed')
+ results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid})
+ (ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n')
+ print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified')
+ finally:command('start','worker')
+if __name__=='__main__':main()