Реализация на 5

This commit is contained in:
Grbisba
2026-11-13 12:48:18 +03:00
parent ce035c8528
commit 2198368a44
60 changed files with 3253 additions and 181 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
.lab5-faults
bin
+6 -4
View File
@@ -30,9 +30,11 @@ RECONCILE_GRACE_PERIOD=1h
# SeaweedFS image and persistent bucket bootstrap are pinned in compose.yaml.
# API has four upload slots and one decoder per replica, GOMEMLIMIT=384MiB.
# Planned lab5 async pipeline settings; the ported lab4 application does not use these yet.
BROKER_URL_FILE=/run/secrets/broker_url
# Async lab5 pipeline. Broker credentials are generated by bootstrap.
BROKER_PASSWORD=replace-with-local-broker-password
WORKER_CONCURRENCY=2
WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
JOB_TIMEOUT=60s
JOB_LEASE=90s
STAGING_RETENTION=24h
+28
View File
@@ -0,0 +1,28 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.27.2'
cache-dependency-path: service/go.sum
- run: make check build vet
- run: make up
- run: make test test-go test-storage test-images test-staging test-broker-persistence
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
- run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
- run: make restore
- run: make test-pipeline
- uses: actions/upload-artifact@v4
if: always()
with:
name: pipeline-evidence
path: evidence/pipeline.json
- run: make down
if: always()
+2
View File
@@ -31,3 +31,5 @@ coverage/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
/.ignore/
.lab5-faults/
+3
View File
@@ -16,6 +16,9 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0
RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0
COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml
COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger
ENV APP_HOST=0.0.0.0 \
+17 -2
View File
@@ -13,8 +13,8 @@ OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen
.PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt
help:
@echo "up/down: lab5 stand (ported lab4 application); build/vet: Go checks"
@echo "test: lab5 API smoke (requires async implementation); test-go/test-storage: existing Go checks"
@echo "up/down: lab5 async image pipeline; build/vet: Go checks"
@echo "test: lab5 API smoke; test-go/test-storage: Go checks"
@echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic"
@echo "check/validate: lab5 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks"
@echo "reconcile/reconcile-apply: storage consistency"
@@ -111,3 +111,18 @@ reconcile:
reconcile-apply:
./scripts/compose.sh run --rm -T operator storage reconcile --apply
.PHONY: test-pipeline
test-pipeline:
$(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json
.PHONY: test-images test-staging
test-images:
$(PYTHON) student-tests/images.py
test-staging:
$(PYTHON) student-tests/storage.py
.PHONY: test-broker-persistence
test-broker-persistence:
$(PYTHON) student-tests/broker_persistence.py
+157 -21
View File
@@ -1,37 +1,173 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: не предоставлены владельцем проекта.
- Целевая оценка: **5**, трек 2 — доказанная идемпотентность versioned pipeline.
- Ревизия: базовый commit и состояние рабочего дерева записываются в `evidence/pipeline.json`; решение пока не закоммичено.
- Использованы Go, pgx, AWS SDK, RabbitMQ `amqp091-go`, ImageMagick;
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Compose и для локальных Go-проверок Go 1.27+. Контейнерная сборка использует Go 1.27.2. Образы закреплены версиями/digest в `compose.yaml` и `Dockerfile`; ImageMagick и модули JPEG/WebP — 7.1.2.35-r0.
## Архитектура и решения
```sh
make check build vet
make up
curl --fail http://localhost:18085/health/ready
make test
make test-go
make test-storage
make test-images test-staging test-broker-persistence
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
make restore
make test-pipeline
make down
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Полная валидация схем:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
```
Bootstrap сохраняет существующие локальные настройки и генерирует синтетические credentials PostgreSQL, Redis, S3 и RabbitMQ. `.env.lab5` и `secrets/lab5` игнорируются Git. Миграции выполняются отдельным одноразовым процессом, bucket создаётся идемпотентно. Внешний endpoint — только `127.0.0.1:18085`; внутренние хранилища портов на host не публикуют. API-реплик по умолчанию две.
`make down` сохраняет volumes. **Намеренное удаление учебных данных**: `./scripts/compose.sh down --volumes`; эта команда не входит в обычные проверки и автоматически не выполняется.
## Архитектура и границы
```mermaid
flowchart LR
Client --> LB[HAProxy]
LB --> API[API x2]
API --> Staging[Private S3 staging]
API --> DB[PostgreSQL: metadata + jobs + outbox]
DB --> Dispatcher
Dispatcher --> Broker[RabbitMQ durable queue]
Broker --> Worker
Worker --> Results[Private S3 versioned results]
Worker --> DB
```
Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Attachment service принимает DTO и reader, проверяет доступ и целостность изображения, управляет транзакцией и состояниями. Репозитории выполняют операции хранения; HTTP, retry-политик и преобразования изображений в SQL/триггерах нет. Канонизация тела сообщения, сравнение идемпотентных запросов и создание Message перенесены в chat service.
До `202` весь проверенный оригинал уже записан в S3 staging, а metadata, job и outbox — в одной завершённой транзакции PostgreSQL. При неопределённом COMMIT нет слепого компенсирующего DELETE. Ответ содержит сохранённый снимок `queued`, даже если быстрый worker уже начал работу.
Staging находится в том же S3-сервисе, что конечные объекты: это разделение prefix и жизненных циклов, а не обещание отложить всю сетевую передачу файла. Проверка корректности изображения при upload включает ограниченное декодирование; resize/crop/WebP и публикация конечного оригинала выполняются отдельно в worker.
API readiness проверяет PostgreSQL и S3. Недоступность broker/worker не блокирует транзакционный приём через outbox. Worker и dispatcher переподключаются после отказа зависимостей. Redis остаётся необязательным кэшем, очередь в него не помещается.
### Инфраструктура
| Роль → service | Версия | Проверка | Volume / сеть / credentials |
| --- | --- | --- | --- |
| API → api, ≥2 | собственная сборка Go 1.27.2 | `/health/ready` | без локального durable state; default + storage; signing/S3/broker files, DB env |
| PostgreSQL → postgres | 17.11, digest в Compose | pg_isready | postgres_data; default; локальный пароль env |
| Кэш → redis | 8.2.11, digest в Compose | authenticated PING | redis_data, AOF; default; локальный пароль env |
| LB → lb | HAProxy 3.2, digest в Compose | backend readiness | без volume данных; default; публичный localhost порт |
| S3 → s3 | SeaweedFS 4.52, digest в Compose | cluster status; bootstrap проверяет запись | s3_data; internal storage; credentials JSON file |
| Broker → broker | RabbitMQ 4.2.0, digest в Compose | rabbitmq-diagnostics check_running | broker_data; default; синтетические credentials |
| Worker → worker | образ приложения + ImageMagick | процесс/reconnect, состояние jobs | эфемерные ограниченные temp files; default + storage; secret files |
| Dispatcher → dispatcher | образ приложения | процесс/reconnect, состояние outbox | без локального durable state; default; broker secret file |
У worker/dispatcher отключён унаследованный HTTP healthcheck: HTTP-сервер они не запускают. Состояние `running` само по себе не подтверждает прогресс; проверять нужно jobs/outbox и end-to-end сценарий.
### Доставка и повторы
- Durable RabbitMQ queue, persistent messages, publisher confirms и mandatory routing. Имя broker фиксировано, чтобы новый контейнер открывал ту же persistent node database; readiness ждёт запуска RabbitMQ application, а не только Erlang node. Consumer ack выполняется после устойчивого результата, постановки следующей попытки либо записи terminal failed.
- Сообщения соответствуют `jobs.schema.json`: UUID, версия, attempt, timestamp, опциональный trace carrier. Байтов файла, URL и credentials в них нет. Worker читает доверенные storage metadata из PostgreSQL.
- `job_id` стабилен для вложения и версии. Redelivery сама не увеличивает attempt; управляемый retry/recovery увеличивает его. По умолчанию пять попыток, exponential backoff с jitter 0.5–30 секунд в зависимости от номера попытки.
- Publisher имеет 20-секундный deadline на самом socket (одного контекста PublishWithContext недостаточно для ограничения сетевой записи). Dispatcher захватывает одну запись с lease 30 секунд, публикует её и фиксирует результат с проверкой token. Просроченный dispatcher не может изменить запись нового владельца. После пяти неудачных публикаций остаётся dead_letter-запись и наблюдаемый failed job.
- Worker: concurrency=2, prefetch=2, timeout=60s, lease=90s. Lease recovery работает независимо от consumer. Неактивный lease не продлевается: работа дольше лимита повторяется ограниченно.
- Постоянная ошибка заканчивается немедленно (`invalid_image`, `source_missing`, `source_corrupt`); временная повторяется и после исчерпания бюджета даёт `retry_exhausted`. API получает только безопасный error_code.
- Реестр failed — PostgreSQL. Replay явный, только для текущего failed job с доступным source; записывается аудит, номер attempt продолжает расти, выдаётся новый бюджет из пяти попыток. Уже успешную job replay не запускает.
```sh
./scripts/compose.sh run --rm -T operator jobs list
./scripts/compose.sh run --rm -T operator jobs replay --id JOB_UUID
./scripts/compose.sh run --rm -T operator outbox replay --id JOB_UUID
./scripts/compose.sh run --rm -T operator jobs reprocess --id ATTACHMENT_UUID --pipeline-version 2
```
`outbox replay` восстанавливает логическую failed job вместе с новой outbox-записью; ID аргумента — job, не отдельная доставка. Старые dead_letter-записи сохраняются для диагностики и не переводят новую попытку обратно в failed.
### Версии и инварианты
Manifest имеет PK `(attachment_id, pipeline_version, variant)`. Варианты публикуются одной DB-транзакцией только после записи всех объектов. Завершение проверяет текущую версию вложения, состояние job, token и действующий lease. Ready не откатывается из-за старой доставки.
Объектные ключи включают вложение, версию, variant и SHA-256. Повтор одинакового эффекта использует тот же ключ; другая версия пишет в свой namespace. Поэтому устаревшая попытка не перезапишет новые байты S3, даже когда БД отклоняет её завершение. Незавершённые объекты возможны и удаляются как сироты после grace; они не становятся опубликованным вторым набором вариантов.
Версия 1 использует WebP quality 80, версия 2 — 85. Остальные параметры одинаковы. Reprocess доступен для ready-вложения и строго большей поддерживаемой версии; на время новой работы статус queued и content=409. Сохраняются прежние manifest, новый оригинал побайтно совпадает с исходником.
### Изображения и cleanup
Thumbnail: JPEG, вписан в 256×256 без увеличения, размеры округляются вниз с минимумом 1 px. Crop: центральный квадрат JPEG 128×128 с разрешённым увеличением. Optimized: WebP внутри 1280×1280 без увеличения. Применяется EXIF orientation, alpha компонуется на белый, метаданные производных удаляются. JPEG quality=85. Не обещается уменьшение каждого файла.
ImageMagick запускается без shell, отдельным дочерним процессом с отменой по timeout. Политика запрещает delegates/filters и разрешает только нужные coders; memory=256MiB, map=0, disk=256MiB, thread=1, time=60s, list-length=4. Worker ограничен 1 CPU/1GiB; его локальные файлы — только восстанавливаемые временные данные. API принимает максимум четыре одновременных upload и выполняет одну проверку декодирования на реплику. Пиксели проверяются до полной распаковки. Превышение ресурсов заканчивается ошибкой, а не бесконечным retry.
Staging удаляется после устойчивого ready; failed source хранится 24h. Источники, которыми являются исторические опубликованные results, физически сохраняются. После удаления failed source replay отклоняется; оператор может восстановить оригинал с проверкой размера/SHA и затем явно повторить job. Reconcile защищает ссылки metadata/manifest и активные jobs, удаляет сироты только после grace=1h (>2×upload timeout). Старые originals lab4 переводятся миграцией на первую версию pipeline без изменения ID и ссылок сообщений.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Фактический вывод команд сохраняется отдельно от описания реализации. `evidence/pipeline.json` содержит повторяемые сценарии, UUID, длительности и результат каждого запуска; `evidence/images.json` — проверки изображений. Проверки защиты staging, удаления после ready, retention failed и восстановления исходника — `evidence/storage.json`. Остальные результаты — `evidence/checks.txt`.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда / наблюдение | Доказательство |
| --- | --- | --- |
| Структура, схемы, сборка и анализ | make check; make validate; make build; make vet | checks.txt |
| Публичный контракт и предыдущие лабораторные | make test; make test-go; make test-storage | checks.txt |
| Сессия и данные после перезапуска; отказ API | test-persistence / test-failover с действительными hooks | checks.txt |
| 202 при остановленном worker, сообщение с queued-вложением, рестарт stores/broker и ready | worker_down_and_store_restart | pipeline.json |
| Опубликованное сообщение переживает пересоздание broker с тем же volume при остановленном dispatcher | make test-broker-persistence; hostname=broker сохраняет имя узла | broker-persistence.json |
| Thumbnail/crop/WebP, orientation, alpha, реальные размеры и hashes | student-tests/images.py | images.json |
| Сбой до/после S3 side effect и до/после COMMIT результата, до ack | crash_before_objects / after_original_put / after_objects / after_result_commit / before_ack | pipeline.json |
| Confirm получен, outbox ещё processing; повтор публикации безопасен | crash_confirm_before_outbox_mark | pipeline.json |
| Ready redelivery — no-op | ready_redelivery, 3 копии | pipeline.json |
| Реальный lease expiry, более новая попытка/версия, возобновление старого worker и поздняя старая доставка | lease_reorder_and_stale_version | pipeline.json |
| Staging защищён при queued; после ready удалён; failed retention и restore проверены | make test-staging | storage.json |
| Retry ограничен, failed наблюдаем, replay явный | bounded_retry_and_explicit_replay / permanent_error | pipeline.json |
| Сбой после staging и разрыв COMMIT/publish | api_crash_after_staging / api_crash_after_upload_commit | pipeline.json |
## Эксперименты и отказы
## Эксперимент трека 2
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Гипотеза:** атомарное сохранение намерения, durable доставка, fencing в БД и разделение версий/контента в S3 сохраняют принятые задания и предотвращают публикацию устаревшего эффекта при проверяемых crash/reorder/duplicate сценариях.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Методика: `make test-pipeline` включает исключительно для эксперимента `compose.faults.yaml`. Директивы в игнорируемом `.lab5-faults/` адресуются по job/attachment ID и точке сбоя. Барьер отмечает достижение точки; crash завершает процесс с кодом 86, restart policy поднимает его снова. Pause применяется один раз: другая попытка может завершиться после реального истечения lease. Общая директория не содержит токенов и очищается при завершении.
## Ограничения и следующие шаги
В эксперименте timeout=5s, lease=8s, concurrency/prefetch=2; остальные параметры не ускоряются искусственным изменением lease в БД. Окно после publisher confirm использует обычный outbox lease=30s. Для гарантированного повторного publish тест повторно делает свою outbox-запись pending; это внесение дубля, а не штатная операция пользователя. Для COMMIT/publish dispatcher заранее остановлен, проверяется pending-запись. Для crash до COMMIT отсутствие `202` не трактуется как потеря принятой работы.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Каждый сценарий выполняется три раза с новыми синтетическими пользователями/вложениями. Основной файл — публичный PNG fixture 320×200, 111817 bytes; дополнительных параллельных upload в fault-эксперименте нет. Измеряется длительность сценария (включает Docker stop/start и проверки), а не чистая latency обработки. Точность таймеров — monotonic clock клиента; нагрузочный максимум этим экспериментом не измеряется.
Окружение выполнения: Apple M4 Pro, 14 CPU, host RAM 24GiB; Docker Desktop arm64, 14 CPU и около 7.75GiB RAM. CPU/RAM контейнеров ограничены Compose. Исследование проводится на одном Docker-host с сохранением volumes, без симуляции потери диска.
### Наблюдавшиеся результаты
Все **39/39** сценариев трека завершились PASS (13 сценариев × 3 повтора). Интервалы ниже включают действия Docker и проверки; это не измерение чистого processing latency.
| Сценарий | Минимум–максимум, с |
| --- | --- |
| worker_down_and_store_restart | 63.864–66.563 |
| crash_before_objects | 10.888–12.269 |
| crash_after_original_put | 11.222–12.171 |
| crash_after_objects | 11.174–11.340 |
| crash_after_result_commit | 1.973–2.088 |
| crash_before_ack | 2.017–2.098 |
| ready_redelivery | 2.435–3.002 |
| crash_confirm_before_outbox_mark | 33.988–36.021 |
| lease_reorder_and_stale_version | 14.652–15.225 |
| bounded_retry_and_explicit_replay | 13.737–15.238 |
| permanent_error | 2.552–2.699 |
| api_crash_after_staging | 0.144–0.817 |
| api_crash_after_upload_commit | 2.141–2.328 |
После эксперимента отдельная проверка retention обнаружила и позволила исправить гонку восстановления исходника: Restore обновляет возраст source, а Cleanup берёт тот же advisory lock. Эта корректировка проверена `make test-staging`. После добавления socket deadline повторно выполнены confirm/crash, lease + stale version и COMMIT/publish на итоговом образе: **3/3 PASS**, см. `evidence/pipeline-final.json`. Image ID исходного полного fault-прогона сохранён в pipeline.json, итогового — в pipeline-final.json.
## Ограничения
At-least-once доставка сохраняется: publish/confirm/COMMIT/ack не образуют одну распределённую транзакцию. Доказывается идемпотентность опубликованного эффекта в перечисленных сценариях, а не exactly-once доставка и не отсутствие любых возможных отказов.
Один PostgreSQL, S3, broker, LB и host — отдельные точки отказа. Durable очередь здесь не означает кластер RabbitMQ. Невосстановимая потеря volumes не покрывается; backup/restore инфраструктуры не реализован. После окончательной ошибки нужен операторский replay; при длительном отказе брокера принятые задания могут стать наблюдаемыми failed. Накопление outbox/jobs не ограничено глобальной квотой: ограничены upload, worker concurrency/prefetch и ресурсы, но отдельный трек перегрузки не заявляется.
Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acks](https://www.rabbitmq.com/docs/confirms), [Go client](https://github.com/rabbitmq/amqp091-go), [ImageMagick resource policy](https://imagemagick.org/resources/).
+22
View File
@@ -0,0 +1,22 @@
# Explicit isolated experiment profile. Never enabled by make up.
services:
api:
environment:
LAB5_FAULTS: "1"
volumes:
- ./.lab5-faults:/faults
worker:
environment:
LAB5_FAULTS: "1"
JOB_TIMEOUT: 5s
JOB_LEASE: 8s
volumes:
- ./.lab5-faults:/faults
dispatcher:
environment:
LAB5_FAULTS: "1"
volumes:
- ./.lab5-faults:/faults
operator:
environment:
LAB5_FAULTS: "1"
+54
View File
@@ -40,6 +40,13 @@ x-api: &api
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
JOB_LEASE: ${JOB_LEASE:-90s}
STAGING_RETENTION: ${STAGING_RETENTION:-24h}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s}
@@ -47,6 +54,7 @@ x-api: &api
GOMEMLIMIT: 384MiB
networks: [default, storage]
volumes:
- ./secrets/lab5/broker_url:/run/secrets/broker_url:ro
- ./secrets/lab5/private_key.pem:/run/certs/private_key.pem:ro
- ./secrets/lab5/public_key.pem:/run/certs/public_key.pem:ro
- ./secrets/lab5/s3_api_access:/run/secrets/s3_access:ro
@@ -78,6 +86,50 @@ services:
depends_on:
postgres:
condition: service_healthy
broker:
hostname: broker
image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524
restart: unless-stopped
cpus: 0.5
mem_limit: 512m
environment:
RABBITMQ_DEFAULT_USER: messenger
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh}
volumes:
- broker_data:/var/lib/rabbitmq
healthcheck:
test: [CMD, rabbitmq-diagnostics, -q, check_running]
interval: 3s
timeout: 5s
retries: 30
stop_grace_period: 30s
worker:
<<: *api
command: [worker]
cpus: 1
mem_limit: 1g
stop_grace_period: 10s
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
broker:
condition: service_healthy
dispatcher:
<<: *api
command: [dispatcher]
cpus: 0.25
mem_limit: 128m
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
lb:
image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0}
restart: unless-stopped
@@ -168,6 +220,7 @@ services:
- ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro
- ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro
- ./tests/fixtures:/fixtures:ro
- ./secrets/lab5/broker_url:/run/secrets/broker_url:ro
healthcheck:
disable: true
depends_on: {}
@@ -176,6 +229,7 @@ volumes:
postgres_data:
redis_data:
s3_data:
broker_data:
networks:
storage:
+10
View File
@@ -0,0 +1,10 @@
{
"result": "PASS",
"attachment_id": "cb1e8982-980d-4286-8224-335b644a5fc5",
"broker_hostname": "broker",
"old_container_id": "16848707eae7bdfb195da43c912ad66399678cc0a48d100bea119619f7e62009",
"new_container_id": "0dfeb1f83d39adc4de9c7798d65b5f125927fe95ae050e4a17e8a5238713b81b",
"message_present_before_and_after": true,
"outbox_published_before_replacement": true,
"dispatcher_stopped_during_replacement": true
}
+14
View File
@@ -0,0 +1,14 @@
{
"date_utc": "2026-11-13T09:29:53.527687+00:00",
"base_commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
"working_tree": "uncommitted solution",
"runtime_source_sha256": "cadfe8fc0d834a598d26e03d1c0348c907ac6fe9a4574aa959f156dfd69488d6",
"hash_method": "Sorted relative path + NUL + file content + NUL, for service/**/*.go, migrations/*.sql, go.mod, go.sum, Dockerfile and ImageMagick policy",
"image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f",
"go_local": "go version go1.27.0 darwin/arm64",
"docker_server": "29.7.2",
"compose": "Docker Compose version v5.3.1",
"docker_resources": "cpus=14 memory_bytes=8320954368 arch=aarch64",
"hardware": "Apple M4 Pro, 14 CPUs, 24 GiB host RAM",
"source_reuse": "Adapted core outbox from supplied mamont-optics project; original project left unchanged"
}
+201
View File
@@ -0,0 +1,201 @@
Lab5 verification, UTC 2026-11-13T09:29:53.527687+00:00
All commands below completed with exit code 0.
Go output excerpts omit SQL, runtime logs and synthetic request bodies.
$ make check build vet; make validate PYTHON=.venv/bin/python
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
cd service && go build ./...
cd service && go build -o ../bin/benchmark ./cmd/benchmark
cd service && go vet ./...
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
$ make test
python3 tests/smoke.py --base-url "http://localhost:18085"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
$ make test-go (race enabled)
? gitea.privcloud.tech/mtusi2026/cmd/app [no test files]
=== RUN TestLostPostResponseIsNotRetried
--- PASS: TestLostPostResponseIsNotRetried (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.461s
? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files]
? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files]
? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files]
=== RUN TestIdempotencyTransactionAndExpiry
PASS
ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.653s
? gitea.privcloud.tech/mtusi2026/internal/config [no test files]
? gitea.privcloud.tech/mtusi2026/internal/dto [no test files]
? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files]
=== RUN TestSessionFingerprints
--- PASS: TestSessionFingerprints (0.02s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.908s
? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files]
=== RUN TestVerifiedSessionIsRequestScoped
--- PASS: TestVerifiedSessionIsRequestScoped (0.00s)
=== RUN TestMissingTokenDoesNotCallSessionStore
--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 2.467s
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files]
? gitea.privcloud.tech/mtusi2026/migrations [no test files]
=== RUN TestInvalidJSON
=== RUN TestInvalidJSON/malformed
=== RUN TestInvalidJSON/wrong_type
=== RUN TestInvalidJSON/unknown_field
--- PASS: TestInvalidJSON (0.01s)
--- PASS: TestInvalidJSON/malformed (0.01s)
--- PASS: TestInvalidJSON/wrong_type (0.00s)
--- PASS: TestInvalidJSON/unknown_field (0.00s)
=== RUN TestForeignChatIsHidden
--- PASS: TestForeignChatIsHidden (0.54s)
=== RUN TestEmptyMessage
--- PASS: TestEmptyMessage (0.21s)
=== RUN TestDuplicateUsername
--- PASS: TestDuplicateUsername (0.28s)
=== RUN TestMessagePagination
--- PASS: TestMessagePagination (0.22s)
=== RUN TestAttachmentContractAndErrors
=== RUN TestAttachmentContractAndErrors/fake
=== RUN TestAttachmentContractAndErrors/mime_mismatch
=== RUN TestAttachmentContractAndErrors/wrong_type
=== RUN TestAttachmentContractAndErrors/truncated
--- PASS: TestAttachmentContractAndErrors (1.05s)
--- PASS: TestAttachmentContractAndErrors/fake (0.00s)
--- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s)
--- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s)
--- PASS: TestAttachmentContractAndErrors/truncated (0.00s)
=== RUN TestConcurrentAttachmentMessage
--- PASS: TestConcurrentAttachmentMessage (0.25s)
=== RUN TestConcurrentMessages
--- PASS: TestConcurrentMessages (1.84s)
=== RUN TestIdempotencyContract
--- PASS: TestIdempotencyContract (0.45s)
=== RUN TestConcurrentIdempotencyAcrossReplicas
--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s)
PASS
ok gitea.privcloud.tech/mtusi2026/tests 7.261s
$ make test-storage (isolated database)
=== RUN TestIdempotencyTransactionAndExpiry
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object
=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock
--- PASS: TestIdempotencyTransactionAndExpiry (0.12s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.02s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s)
PASS
$ make test-images
PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs
$ make test-staging
PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified
$ make test-broker-persistence
PASS: confirmed message survived broker container replacement with the same volume and stable node identity
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.273}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
Actual process state before persistence hook
/mtusi2026lab5-api-1 pid=63585 started=2026-11-13T09:13:37.287792805Z
/mtusi2026lab5-api-2 pid=63443 started=2026-11-13T09:13:37.185017014Z
/mtusi2026lab5-broker-1 pid=16156 started=2026-11-13T09:06:10.180568751Z
/mtusi2026lab5-dispatcher-1 pid=63444 started=2026-11-13T09:13:37.187137055Z
/mtusi2026lab5-lb-1 pid=6588 started=2026-11-13T08:49:29.974451052Z
/mtusi2026lab5-postgres-1 pid=15734 started=2026-11-13T09:06:09.241419334Z
/mtusi2026lab5-redis-1 pid=15771 started=2026-11-13T09:06:09.289178667Z
/mtusi2026lab5-s3-1 pid=17699 started=2026-11-13T09:06:34.353438762Z
/mtusi2026lab5-worker-1 pid=66553 started=2026-11-13T09:14:11.339788001Z
Actual process state after persistence hook
/mtusi2026lab5-api-1 pid=73635 started=2026-11-13T09:15:52.601737132Z
/mtusi2026lab5-api-2 pid=73510 started=2026-11-13T09:15:52.490649465Z
/mtusi2026lab5-broker-1 pid=70234 started=2026-11-13T09:14:51.740755173Z
/mtusi2026lab5-dispatcher-1 pid=73512 started=2026-11-13T09:15:52.489731548Z
/mtusi2026lab5-lb-1 pid=73700 started=2026-11-13T09:15:52.696327173Z
/mtusi2026lab5-postgres-1 pid=70113 started=2026-11-13T09:14:50.775761506Z
/mtusi2026lab5-redis-1 pid=70044 started=2026-11-13T09:14:50.718397506Z
/mtusi2026lab5-s3-1 pid=71476 started=2026-11-13T09:15:16.065232545Z
/mtusi2026lab5-worker-1 pid=73511 started=2026-11-13T09:15:52.490188673Z
Actual state after API failover, before restoring stopped replica
api mtusi2026lab5-api-1 running healthy
api mtusi2026lab5-api-2 exited
broker mtusi2026lab5-broker-1 running healthy
dispatcher mtusi2026lab5-dispatcher-1 running
lb mtusi2026lab5-lb-1 running healthy
migrate mtusi2026lab5-migrate-1 exited
postgres mtusi2026lab5-postgres-1 running healthy
redis mtusi2026lab5-redis-1 running healthy
s3 mtusi2026lab5-s3-1 running healthy
s3-init mtusi2026lab5-s3-init-1 exited
worker mtusi2026lab5-worker-1 running
Effective container resource limits
/mtusi2026lab5-api-1 memory_bytes=536870912 cpu_nano=1000000000
/mtusi2026lab5-api-2 memory_bytes=536870912 cpu_nano=1000000000
/mtusi2026lab5-broker-1 memory_bytes=536870912 cpu_nano=500000000
/mtusi2026lab5-dispatcher-1 memory_bytes=134217728 cpu_nano=250000000
/mtusi2026lab5-lb-1 memory_bytes=134217728 cpu_nano=500000000
/mtusi2026lab5-postgres-1 memory_bytes=1073741824 cpu_nano=2000000000
/mtusi2026lab5-redis-1 memory_bytes=134217728 cpu_nano=250000000
/mtusi2026lab5-s3-1 memory_bytes=536870912 cpu_nano=1000000000
/mtusi2026lab5-worker-1 memory_bytes=1073741824 cpu_nano=1000000000
make test-pipeline: 39/39 PASS; details and repetitions in pipeline.json
Final image targeted fault checks: 3/3 PASS; details in pipeline-final.json
+53
View File
@@ -0,0 +1,53 @@
{
"results": [
{
"case": "small",
"result": "PASS",
"thumbnail": [
16,
8
]
},
{
"case": "floor",
"result": "PASS",
"thumbnail": [
256,
76
]
},
{
"case": "alpha",
"result": "PASS",
"thumbnail": [
40,
20
]
},
{
"case": "orientation",
"result": "PASS",
"thumbnail": [
128,
256
]
},
{
"case": "thin",
"result": "PASS",
"thumbnail": [
1,
256
]
},
{
"case": "center_crop",
"result": "PASS",
"thumbnail": [
256,
85
]
}
],
"negative_cases": 4
}
+42
View File
@@ -0,0 +1,42 @@
{
"date_utc": "2026-11-13T09:24:08.989020+00:00",
"commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
"working_tree": "implementation changes uncommitted",
"repeats": 1,
"worker": {
"concurrency": 2,
"prefetch": 2,
"timeout_seconds": 5,
"lease_seconds": 8,
"memory": "1GiB",
"cpus": 1
},
"results": [
{
"scenario": "crash_confirm_before_outbox_mark",
"run": 1,
"result": "PASS",
"seconds": 33.527,
"job_id": "ce8a7658-c069-429a-b655-cb051f922e14"
},
{
"scenario": "lease_reorder_and_stale_version",
"run": 1,
"result": "PASS",
"seconds": 14.623,
"attachment_id": "6e709809-9ca2-4fb8-b7f6-155a3aace89c",
"old_job_id": "421fa4dc-3704-4f87-a490-3110f062459a",
"pipeline_version": 2,
"manifest_rows": 8
},
{
"scenario": "api_crash_after_upload_commit",
"run": 1,
"result": "PASS",
"seconds": 2.122,
"attachment_id": "293e92d6-4253-4fd9-8bad-719a488f6bdf",
"accepted_commit": true
}
],
"image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f"
}
+337
View File
@@ -0,0 +1,337 @@
{
"date_utc": "2026-11-13T09:09:05.756165+00:00",
"commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
"working_tree": "implementation changes uncommitted",
"repeats": 3,
"worker": {
"concurrency": 2,
"prefetch": 2,
"timeout_seconds": 5,
"lease_seconds": 8,
"memory": "1GiB",
"cpus": 1
},
"results": [
{
"scenario": "worker_down_and_store_restart",
"run": 1,
"result": "PASS",
"seconds": 66.563,
"attachment_id": "912b15f8-2ddb-4cbc-8db7-526540256749",
"job_id": "4913bc34-1533-4140-a0dc-c53efa5be0dd",
"postgres_restarted": true
},
{
"scenario": "crash_before_objects",
"run": 1,
"result": "PASS",
"seconds": 10.888,
"job_id": "bfdb496b-f2e0-4836-9559-1a9b07ed21c4",
"manifest_rows": 4
},
{
"scenario": "crash_after_original_put",
"run": 1,
"result": "PASS",
"seconds": 12.171,
"job_id": "8468e628-5eef-42c2-9a86-d050d74cb672",
"manifest_rows": 4
},
{
"scenario": "crash_after_objects",
"run": 1,
"result": "PASS",
"seconds": 11.174,
"job_id": "4c587b0f-fa79-4d9e-a3c7-4904db1e80d3",
"manifest_rows": 4
},
{
"scenario": "crash_after_result_commit",
"run": 1,
"result": "PASS",
"seconds": 1.973,
"job_id": "419c9eeb-6467-4138-b05e-92f1bc929df8",
"manifest_rows": 4
},
{
"scenario": "crash_before_ack",
"run": 1,
"result": "PASS",
"seconds": 2.017,
"job_id": "face4c1b-5597-4b24-8f61-a0e756e601de",
"manifest_rows": 4
},
{
"scenario": "ready_redelivery",
"run": 1,
"result": "PASS",
"seconds": 2.435,
"job_id": "514c616d-367c-4693-8d33-60079b2ced87",
"copies": 3
},
{
"scenario": "crash_confirm_before_outbox_mark",
"run": 1,
"result": "PASS",
"seconds": 33.996,
"job_id": "71f7c962-009e-4982-ae7b-e7f24fdcdc7f"
},
{
"scenario": "lease_reorder_and_stale_version",
"run": 1,
"result": "PASS",
"seconds": 14.652,
"attachment_id": "52a2e241-60a1-4443-8ef7-2e8e3e860577",
"old_job_id": "1c5824ca-774e-4230-9764-3e0938f4efeb",
"pipeline_version": 2,
"manifest_rows": 8
},
{
"scenario": "bounded_retry_and_explicit_replay",
"run": 1,
"result": "PASS",
"seconds": 14.47,
"job_id": "8b8f8975-f4c8-45cf-80e9-00626c4e5c36",
"failed_attempt": 5,
"replayed_attempt": 6
},
{
"scenario": "permanent_error",
"run": 1,
"result": "PASS",
"seconds": 2.699,
"job_id": "e1948ce8-b194-4f66-8008-6e733ad42959"
},
{
"scenario": "api_crash_after_staging",
"run": 1,
"result": "PASS",
"seconds": 0.817,
"attachment_id": "2d831ec0-5918-4290-8e26-3c2bec3cc0fd",
"accepted_commit": false
},
{
"scenario": "api_crash_after_upload_commit",
"run": 1,
"result": "PASS",
"seconds": 2.328,
"attachment_id": "f0da85c6-003c-4d79-ba01-f3f961064741",
"accepted_commit": true
},
{
"scenario": "worker_down_and_store_restart",
"run": 2,
"result": "PASS",
"seconds": 66.013,
"attachment_id": "64517fe4-6419-4257-bbf5-a47a88b51068",
"job_id": "873b9510-7ada-4dc0-bdfb-3e204ae2bc3d",
"postgres_restarted": true
},
{
"scenario": "crash_before_objects",
"run": 2,
"result": "PASS",
"seconds": 11.241,
"job_id": "1c0ea968-87d2-45df-bb9b-d8b45c68dc1d",
"manifest_rows": 4
},
{
"scenario": "crash_after_original_put",
"run": 2,
"result": "PASS",
"seconds": 11.222,
"job_id": "748017a5-8f09-4741-8e35-35e4dbcb3c55",
"manifest_rows": 4
},
{
"scenario": "crash_after_objects",
"run": 2,
"result": "PASS",
"seconds": 11.34,
"job_id": "489d2edd-b008-48fc-ba35-c9f64ee05ba2",
"manifest_rows": 4
},
{
"scenario": "crash_after_result_commit",
"run": 2,
"result": "PASS",
"seconds": 2.088,
"job_id": "0ed57d8a-3fcc-42a4-a31f-2f02ae4f5f5f",
"manifest_rows": 4
},
{
"scenario": "crash_before_ack",
"run": 2,
"result": "PASS",
"seconds": 2.026,
"job_id": "e9a8270c-3132-460a-a888-ee8722798e35",
"manifest_rows": 4
},
{
"scenario": "ready_redelivery",
"run": 2,
"result": "PASS",
"seconds": 2.704,
"job_id": "b4f7d69b-3de5-4350-97bf-0e880ed94b0e",
"copies": 3
},
{
"scenario": "crash_confirm_before_outbox_mark",
"run": 2,
"result": "PASS",
"seconds": 33.988,
"job_id": "5f7a89af-a045-423c-8c70-47b19615b577"
},
{
"scenario": "lease_reorder_and_stale_version",
"run": 2,
"result": "PASS",
"seconds": 14.694,
"attachment_id": "a1230ca9-8001-4bb4-b648-51446f646cbf",
"old_job_id": "2f1e1fbf-856d-4605-873c-b23011072a4e",
"pipeline_version": 2,
"manifest_rows": 8
},
{
"scenario": "bounded_retry_and_explicit_replay",
"run": 2,
"result": "PASS",
"seconds": 13.737,
"job_id": "737209e2-5d3c-4693-8394-fc8173b88ee2",
"failed_attempt": 5,
"replayed_attempt": 6
},
{
"scenario": "permanent_error",
"run": 2,
"result": "PASS",
"seconds": 2.552,
"job_id": "e5e6dc90-9ea5-4d7c-aa11-c6c275d69793"
},
{
"scenario": "api_crash_after_staging",
"run": 2,
"result": "PASS",
"seconds": 0.153,
"attachment_id": "ed8e2936-47fa-4717-9eb3-e63292c8a0ad",
"accepted_commit": false
},
{
"scenario": "api_crash_after_upload_commit",
"run": 2,
"result": "PASS",
"seconds": 2.159,
"attachment_id": "a739f237-2278-421c-aa55-6024e717ad89",
"accepted_commit": true
},
{
"scenario": "worker_down_and_store_restart",
"run": 3,
"result": "PASS",
"seconds": 63.864,
"attachment_id": "95176e6d-301a-4725-9861-7133507184eb",
"job_id": "e5f8b9fa-e460-4441-92b0-2af2ac2540bf",
"postgres_restarted": true
},
{
"scenario": "crash_before_objects",
"run": 3,
"result": "PASS",
"seconds": 12.269,
"job_id": "67c70b6e-ea42-4c1d-894c-d2234a70bf03",
"manifest_rows": 4
},
{
"scenario": "crash_after_original_put",
"run": 3,
"result": "PASS",
"seconds": 11.402,
"job_id": "386f9b0d-1b1f-42b6-95a0-a13d6b97f118",
"manifest_rows": 4
},
{
"scenario": "crash_after_objects",
"run": 3,
"result": "PASS",
"seconds": 11.259,
"job_id": "9c639085-f040-4989-90ba-54376c64630d",
"manifest_rows": 4
},
{
"scenario": "crash_after_result_commit",
"run": 3,
"result": "PASS",
"seconds": 2.068,
"job_id": "73fe51b1-32eb-4c93-b8b0-0ed920a6641d",
"manifest_rows": 4
},
{
"scenario": "crash_before_ack",
"run": 3,
"result": "PASS",
"seconds": 2.098,
"job_id": "2947da70-b658-48c0-9aa2-905bc0927781",
"manifest_rows": 4
},
{
"scenario": "ready_redelivery",
"run": 3,
"result": "PASS",
"seconds": 3.002,
"job_id": "4f67af5a-1014-4f91-9a68-54739f2edde7",
"copies": 3
},
{
"scenario": "crash_confirm_before_outbox_mark",
"run": 3,
"result": "PASS",
"seconds": 36.021,
"job_id": "c4135c71-64e7-44ba-9e4f-bf8f7489a459"
},
{
"scenario": "lease_reorder_and_stale_version",
"run": 3,
"result": "PASS",
"seconds": 15.225,
"attachment_id": "3bdce3c7-d7d5-49fb-9691-96e626317ae1",
"old_job_id": "dd3d35b5-14fe-420b-a650-e740bc43489c",
"pipeline_version": 2,
"manifest_rows": 8
},
{
"scenario": "bounded_retry_and_explicit_replay",
"run": 3,
"result": "PASS",
"seconds": 15.238,
"job_id": "444a1bf1-42ed-4123-9ec7-7f614cd6d90f",
"failed_attempt": 5,
"replayed_attempt": 6
},
{
"scenario": "permanent_error",
"run": 3,
"result": "PASS",
"seconds": 2.604,
"job_id": "d21ee0e0-702d-4459-8fd6-5e5f62c6c11d"
},
{
"scenario": "api_crash_after_staging",
"run": 3,
"result": "PASS",
"seconds": 0.144,
"attachment_id": "963d2034-021f-492f-81f3-d31431a2bc0f",
"accepted_commit": false
},
{
"scenario": "api_crash_after_upload_commit",
"run": 3,
"result": "PASS",
"seconds": 2.141,
"attachment_id": "ea757ed8-98a8-4f3b-993e-556347380e26",
"accepted_commit": true
}
],
"tested_image_id": "sha256:0b720b9bab11e4a762e8b180b26c290a43503c4db2db3436c201cb08725f24c0",
"runtime_source_sha256": "e6460140119a22471f992cd02c721b55463d0810a6a8e50333e193d674c7c79a",
"subsequent_fix": "Restore refreshes retention timestamp; Cleanup shares its source lock. Publisher has a physical socket deadline. Verified by storage.py and the additional final-image fault scenarios in pipeline-final.json; job processing, lease and manifest logic unchanged."
}
+24
View File
@@ -0,0 +1,24 @@
{
"results": [
{
"scenario": "queued_staging_protected",
"result": "PASS",
"attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b"
},
{
"scenario": "staging_deleted_after_ready",
"result": "PASS",
"attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b"
},
{
"scenario": "failed_retention_and_missing_source_replay",
"result": "PASS",
"age_simulated_hours": 25
},
{
"scenario": "verified_restore_then_replay",
"result": "PASS",
"job_id": "8d6f4591-e10f-4616-b387-fe7ec5020a84"
}
]
}
+13
View File
@@ -0,0 +1,13 @@
<policymap>
<policy domain="delegate" rights="none" pattern="*"/>
<policy domain="filter" rights="none" pattern="*"/>
<policy domain="coder" rights="none" pattern="*"/>
<policy domain="coder" rights="read|write" pattern="{PNG,JPEG,WEBP,NULL}"/>
<policy domain="resource" name="memory" value="256MiB"/>
<policy domain="resource" name="map" value="0"/>
<policy domain="resource" name="disk" value="256MiB"/>
<policy domain="resource" name="thread" value="1"/>
<policy domain="resource" name="time" value="60"/>
<policy domain="resource" name="list-length" value="4"/>
<policy domain="path" rights="none" pattern="@*"/>
</policymap>
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""Create synthetic local broker credentials, preserving existing settings."""
from pathlib import Path
import secrets
from urllib.parse import quote
root=Path(__file__).resolve().parents[1]
env=root/'.env.lab5'
values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#'))
password=values.get('BROKER_PASSWORD')
if not password:
password=secrets.token_hex(24)
with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n')
p=root/'secrets/lab5/broker_url'
if p.exists(): p.chmod(0o600)
p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n')
p.chmod(0o444)
print('Local broker credentials ready.')
+2
View File
@@ -21,3 +21,5 @@ chmod 444 secrets/lab5/*.pem
printf '%s\n' 'Local lab5 settings and synthetic keys ready; existing files preserved.'
python3 scripts/bootstrap-storage.py
python3 scripts/bootstrap-broker.py
+3 -3
View File
@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
./scripts/compose.sh stop api lb
./scripts/compose.sh restart postgres redis s3
./scripts/compose.sh stop api lb worker dispatcher
./scripts/compose.sh restart postgres redis s3 broker
./scripts/compose.sh run --rm -T s3-init
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher
+10
View File
@@ -27,6 +27,7 @@ import (
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/auth"
"gitea.privcloud.tech/mtusi2026/internal/service/chat"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"gitea.privcloud.tech/mtusi2026/internal/service/user"
"github.com/google/uuid"
stdhttp "net/http"
@@ -56,6 +57,9 @@ func run() error {
if err != nil {
return err
}
if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") {
return runPipeline(cfg, attachmentConfig, os.Args[1:])
}
if len(os.Args) > 1 && os.Args[1] == "storage" {
return runStorage(cfg, attachmentConfig, os.Args[2:])
}
@@ -91,6 +95,12 @@ func run() error {
chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache)
objectStore := objectstore.New(attachmentConfig)
attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc)
pipelineConfig, err := config.LoadPipeline()
if err != nil {
return err
}
outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil)
attachmentSvc.Configure(tm, outboxSvc, pipelineConfig)
httpRouter := router.New(
middleware.NewAuthMiddleware(authSvc),
+138
View File
@@ -0,0 +1,138 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"gitea.privcloud.tech/mtusi2026/internal/adapter/broker"
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
)
func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error {
c, e := config.LoadPipeline()
if e != nil {
return e
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
pool := postgres.New(cfg.Postgres)
defer pool.Close()
tm := txmanager.New(pool)
repo := &storage.AttachmentsRepo{Pool: pool}
rabbit := &broker.Rabbit{URL: c.BrokerURL}
ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit)
svc := attachment.New(repo, objectstore.New(a), a, nil)
svc.Configure(tm, ob, c)
ob.OnDead = svc.PublishFailed
if args[0] == "jobs" || args[0] == "outbox" {
if len(args) < 2 {
return errors.New("operator action required")
}
fs := flag.NewFlagSet(args[1], flag.ContinueOnError)
idText := fs.String("id", "", "job ID (attachment ID for reprocess)")
version := fs.Int("pipeline-version", 2, "new pipeline version")
if e = fs.Parse(args[2:]); e != nil {
return e
}
if args[1] == "list" {
jobs, e := repo.ListJobs(ctx)
if e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(jobs)
}
id, e := uuid.Parse(*idText)
if e != nil {
return e
}
switch args[1] {
case "redeliver":
if os.Getenv("LAB5_FAULTS") != "1" {
return errors.New("redeliver is restricted to the explicit fault profile")
}
var payload []byte
e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
j, e := repo.JobTx(ctx, tx, id)
if e != nil {
return e
}
payload, e = json.Marshal(j.Message)
return e
})
if e == nil {
e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload})
}
case "replay":
e = svc.Replay(ctx, id)
case "reprocess":
e = svc.Reprocess(ctx, id, *version)
default:
return errors.New("unknown operator action")
}
if e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id})
}
if c.BrokerURL == "" {
return errors.New("BROKER_URL_FILE is required")
}
if args[0] == "worker" {
// Recovery continues while the consumer is disconnected or a delivery is paused.
recoveryDone := make(chan struct{})
defer func() { stop(); <-recoveryDone }()
go func() {
defer close(recoveryDone)
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if e := svc.Recover(ctx); e != nil {
slog.Warn("job_recovery_failed")
}
if e := svc.Cleanup(ctx); e != nil {
slog.Warn("source_cleanup_failed")
}
}
}
}()
for ctx.Err() == nil {
if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil {
slog.Warn("consumer_reconnecting")
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
for ctx.Err() == nil {
_, e = ob.ProcessOnce(ctx)
if e != nil {
slog.Warn("dispatcher_retry")
}
select {
case <-ctx.Done():
case <-time.After(100 * time.Millisecond):
}
}
return nil
}
+8
View File
@@ -20,7 +20,9 @@ import (
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
)
func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
@@ -93,6 +95,12 @@ func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
pool := postgres.New(cfg.Postgres)
defer pool.Close()
svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil)
jobs, e := config.LoadPipeline()
if e != nil {
return e
}
tm := txmanager.New(pool)
svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs)
switch args[0] {
case "reconcile":
fs := flag.NewFlagSet("reconcile", flag.ContinueOnError)
+1
View File
@@ -52,6 +52,7 @@ require (
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
+2
View File
@@ -94,6 +94,8 @@ github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/Ge
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE=
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+170
View File
@@ -0,0 +1,170 @@
package broker
import (
"bytes"
"context"
"encoding/json"
"errors"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/pkg/fault"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"github.com/google/uuid"
amqp "github.com/rabbitmq/amqp091-go"
"io"
"log/slog"
"net"
"regexp"
"time"
)
const Queue = "attachments.v1"
var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
type Rabbit struct{ URL string }
func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) {
dial := amqp.DefaultDial(5 * time.Second)
if deadline > 0 {
dial = func(network, address string) (net.Conn, error) {
c, e := net.DialTimeout(network, address, 5*time.Second)
if e != nil {
return nil, e
}
if e = c.SetDeadline(time.Now().Add(deadline)); e != nil {
c.Close()
return nil, e
}
return c, nil
}
}
conn, e := amqp.DialConfig(r.URL, amqp.Config{Heartbeat: 10 * time.Second, Dial: dial})
if e != nil {
return nil, nil, errors.New("broker connection unavailable")
}
ch, e := conn.Channel()
if e != nil {
conn.Close()
return nil, nil, errors.New("broker channel unavailable")
}
_, e = ch.QueueDeclare(Queue+".invalid", true, false, false, false, nil)
if e == nil {
e = ch.ExchangeDeclare("attachments.invalid", "direct", true, false, false, false, nil)
}
if e == nil {
e = ch.QueueBind(Queue+".invalid", "invalid", "attachments.invalid", false, nil)
}
if e == nil {
_, e = ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{"x-dead-letter-exchange": "attachments.invalid", "x-dead-letter-routing-key": "invalid"})
}
if e != nil {
ch.Close()
conn.Close()
return nil, nil, errors.New("broker topology unavailable")
}
return conn, ch, nil
}
func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) error {
conn, ch, e := r.connect(20 * time.Second)
if e != nil {
return e
}
defer conn.Close()
defer ch.Close()
if e = ch.Confirm(false); e != nil {
return e
}
confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1))
returns := ch.NotifyReturn(make(chan amqp.Return, 1))
var job dto.JobMessage
if e = json.Unmarshal(m.Payload, &job); e != nil {
return e
}
fault.Hit(ctx, "before_publish", job.JobID)
if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: m.Payload}); e != nil {
return errors.New("broker publish failed")
}
select {
case c, ok := <-confirms:
if !ok || !c.Ack {
return errors.New("broker did not confirm")
}
select {
case <-returns:
return errors.New("unroutable job")
default:
}
case <-time.After(15 * time.Second):
return errors.New("confirm timeout")
case <-ctx.Done():
return ctx.Err()
}
fault.Hit(ctx, "after_publish", job.JobID)
return nil
}
func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error {
conn, ch, e := r.connect(0)
if e != nil {
return e
}
defer conn.Close()
defer ch.Close()
if e = ch.Qos(prefetch, 0, false); e != nil {
return e
}
deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil)
if e != nil {
return e
}
done := make(chan struct{})
go func() {
select {
case <-ctx.Done():
conn.Close()
case <-done:
}
}()
defer close(done)
tasks := make(chan amqp.Delivery)
finished := make(chan struct{}, concurrency)
for range concurrency {
go func() {
defer func() { finished <- struct{}{} }()
for d := range tasks {
var m dto.JobMessage
dec := json.NewDecoder(bytes.NewReader(d.Body))
dec.DisallowUnknownFields()
e := dec.Decode(&m)
if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) {
_ = d.Reject(false)
continue
}
if e = process(ctx, m); e != nil {
slog.Warn("job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable")
select {
case <-ctx.Done():
case <-time.After(200 * time.Millisecond):
}
_ = d.Nack(false, true)
continue
}
fault.Hit(ctx, "before_ack", m.JobID)
_ = d.Ack(false)
}
}()
}
for d := range deliveries {
select {
case tasks <- d:
case <-ctx.Done():
}
}
close(tasks)
for range concurrency {
<-finished
}
if ctx.Err() != nil {
return ctx.Err()
}
return errors.New("broker consumer disconnected")
}
@@ -1,12 +1,14 @@
package http
import (
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
"gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider"
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework"
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"io"
"mime"
"net/http"
"strconv"
"strings"
@@ -41,7 +43,20 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename"))
return
}
a, e := h.svc.Upload(w, r, user.UserID, id, values[0])
if !attachment.ValidFilename(values[0]) {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename"))
return
}
mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
if e != nil || (mt != "image/png" && mt != "image/jpeg") {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG"))
return
}
if r.ContentLength > h.svc.Config.MaxBytes {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large"))
return
}
a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)})
if e != nil {
if apperr.CodeFromError(e) == apperr.CodeTooManyRequests {
w.Header().Set("Retry-After", "1")
@@ -49,7 +64,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
framework.JSONError(r.Context(), w, e)
return
}
framework.JSONResponse(w, http.StatusCreated, a)
framework.JSONResponse(w, http.StatusAccepted, a)
}
func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
user, e := framework.ParseToken(r, h.provider)
@@ -61,27 +76,32 @@ func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
if e != nil {
return
}
a, e := h.svc.Get(r.Context(), user.UserID, id)
if !strings.HasSuffix(r.URL.Path, "/content") {
a, e := h.svc.Get(r.Context(), user.UserID, id)
if e != nil {
framework.JSONError(r.Context(), w, e)
return
}
framework.JSONResponse(w, 200, a.Public)
return
}
v := r.URL.Query().Get("variant")
if v == "" {
v = "original"
}
if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant"))
return
}
obj, e := h.svc.Content(r.Context(), user.UserID, id, v)
if e != nil {
framework.JSONError(r.Context(), w, e)
return
}
if !strings.HasSuffix(r.URL.Path, "/content") {
framework.JSONResponse(w, 200, a.Public)
return
}
if v := r.URL.Query().Get("variant"); v != "" && v != "original" {
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant"))
return
}
obj, e := h.svc.Store.Get(r.Context(), a.Key)
if e != nil {
framework.JSONError(r.Context(), w, attachment.Unavailable())
return
}
defer obj.Body.Close()
w.Header().Set("Content-Type", string(a.Public.ContentType))
w.Header().Set("Content-Length", strconv.Itoa(a.Public.SizeBytes))
w.Header().Set("Content-Type", obj.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
// Once streaming has started a transport failure terminates the response; no JSON
// error can be appended to an image. Content-Length exposes incomplete downloads.
@@ -25,10 +25,15 @@ func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*Attachm
}
func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) {
var found bool
e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)", key).Scan(&found)
e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found)
return found, e
}
func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) {
tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing)
return tag.RowsAffected() == 1, e
}
func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error {
_, e := c.Exec(ctx, "SELECT 1")
return e
}
@@ -8,40 +8,34 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
"encoding/json"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
)
type AttachmentRecord struct {
Public contracts.Attachment
Key string
Missing bool
}
type AttachmentRecord = dto.AttachmentRecord
type AttachmentsRepo struct{ Pool *pgxpool.Pool }
const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing"
const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys"
func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) {
a := new(AttachmentRecord)
p := &a.Public
if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing); err != nil {
var variants, keys []byte
if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil {
return nil, apperr.WrapPostgres(err, "attachment unavailable")
}
p.Status = "ready"
p.Variants = []contracts.Variant{}
if err := json.Unmarshal(variants, &p.Variants); err != nil {
return nil, err
}
if err := json.Unmarshal(keys, &a.VariantKeys); err != nil {
return nil, err
}
return a, nil
}
func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) {
return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id))
}
func (r *AttachmentsRepo) Save(ctx context.Context, conn *pgxpool.Conn, a *AttachmentRecord) error {
p := a.Public
_, e := conn.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.Key)
if e != nil {
return apperr.WrapPostgres(e, "failed to save attachment")
}
return nil
}
// Session locks span S3 work and the database commit. A lost connection releases
// the lock; the mandatory grace period also exceeds the maximum upload lifetime.
@@ -58,7 +58,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil {
if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil {
t.Fatal(err)
}
if err = tx.Rollback(ctx); err != nil {
@@ -75,7 +75,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
first, err := repo.CreateIdempotentTx(ctx, tx, req)
first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req)
if err != nil {
t.Fatal(err)
}
@@ -91,7 +91,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
second, err := repo.CreateIdempotentTx(ctx, tx, req)
second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req)
if err != nil {
t.Fatal(err)
}
@@ -115,7 +115,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err = repo.CreateIdempotentTx(ctx, tx, req); err == nil {
if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil {
t.Fatal("expected injected failure")
}
if err = tx.Rollback(ctx); err != nil {
@@ -134,7 +134,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil {
if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil {
t.Fatal("transient failure was cached", err)
}
if err = tx.Commit(ctx); err != nil {
@@ -257,7 +257,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = repo.CreateIdempotentTx(ctx, tx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}})
_, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}})
if err == nil {
t.Fatal("expected missing attachment FK failure")
}
@@ -0,0 +1,134 @@
package storage
import (
"context"
"encoding/json"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"time"
)
func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) }
func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) {
return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id))
}
func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
p := a.Public
_, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status)
return e
}
func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
variants, e := json.Marshal(a.Public.Variants)
if e != nil {
return e
}
keys, e := json.Marshal(a.VariantKeys)
if e != nil {
return e
}
_, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing)
return e
}
const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at`
func scanJob(row pgx.Row) (*dto.AttachmentJob, error) {
j := new(dto.AttachmentJob)
m := &j.Message
err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt)
m.SchemaVersion = 1
m.Type = "attachment.process.v1"
return j, err
}
func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) {
return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id))
}
func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
m := j.Message
_, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt)
return e
}
func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
_, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt)
return e
}
func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) {
rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until LIMIT 100`)
if e != nil {
return nil, e
}
defer rows.Close()
var ids []uuid.UUID
for rows.Next() {
var id uuid.UUID
if e = rows.Scan(&id); e != nil {
return nil, e
}
ids = append(ids, id)
}
return ids, rows.Err()
}
func (r *AttachmentsRepo) AuditTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, action string) error {
_, e := tx.Exec(ctx, `INSERT INTO job_audit(job_id,action) VALUES($1,$2)`, id, action)
return e
}
func (r *AttachmentsRepo) ListJobs(ctx context.Context) ([]dto.AttachmentJob, error) {
rows, e := r.Pool.Query(ctx, `SELECT `+jobColumns+` FROM attachment_jobs ORDER BY updated_at DESC LIMIT 100`)
if e != nil {
return nil, e
}
defer rows.Close()
out := []dto.AttachmentJob{}
for rows.Next() {
j, e := scanJob(rows)
if e != nil {
return nil, e
}
out = append(out, *j)
}
return out, rows.Err()
}
func (r *AttachmentsRepo) FailedOutbox(ctx context.Context) ([]uuid.UUID, error) {
rows, e := r.Pool.Query(ctx, `SELECT DISTINCT o.job_id FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE o.status='dead_letter' AND j.state='queued' AND (o.payload->>'attempt')::integer=j.attempt LIMIT 100`)
if e != nil {
return nil, e
}
defer rows.Close()
var ids []uuid.UUID
for rows.Next() {
var id uuid.UUID
if e = rows.Scan(&id); e != nil {
return nil, e
}
ids = append(ids, id)
}
return ids, rows.Err()
}
func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) {
rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before)
if e != nil {
return nil, e
}
defer rows.Close()
var out []*dto.AttachmentRecord
for rows.Next() {
a, e := ScanAttachment(rows)
if e != nil {
return nil, e
}
out = append(out, a)
}
return out, rows.Err()
}
func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error {
_, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size)
return e
}
func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) {
var dead bool
e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead)
return dead, e
}
@@ -1,10 +1,8 @@
package storage
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"time"
@@ -26,55 +24,49 @@ func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo {
return &MessagesRepo{pool: pool}
}
// CreateIdempotentTx serializes only requests sharing an idempotency scope.
// The transaction commits both the message and its replay record, or neither.
func (r *MessagesRepo) CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) {
scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil {
return nil, apperr.WrapPostgres(err, "failed to lock message key")
// LockKeyTx serializes persistence for one idempotency scope.
func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error {
_, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope)
if err != nil {
return apperr.WrapPostgres(err, "failed to lock message key")
}
hash := messageHash(req)
var hashVersion int
var storedHash []byte
var messageID uuid.UUID
err := tx.QueryRow(ctx, `SELECT request_hash, message_id, hash_version FROM message_idempotency
WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`,
req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&storedHash, &messageID, &hashVersion)
if err == nil {
if hashVersion == 1 && len(req.AttachmentIDs) == 0 {
hash = sha256.Sum256([]byte(req.Text))
} else if hashVersion != 2 {
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
}
if !bytes.Equal(hash[:], storedHash) {
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
}
return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, messageID))
return nil
}
func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) {
v := new(dto.MessageReplay)
err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
if err != nil {
return nil, apperr.WrapPostgres(err, "failed to read message key")
}
message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: time.Now().UTC().Truncate(time.Microsecond)}
if _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`,
message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt); err != nil {
return v, nil
}
func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) {
return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id))
}
func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error {
_, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt)
if err != nil {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation {
return nil, apperr.Wrap(err, apperr.CodeNotFound, "chat not found")
return apperr.Wrap(err, apperr.CodeNotFound, "chat not found")
}
return nil, apperr.WrapPostgres(err, "failed to create message")
return apperr.WrapPostgres(err, "failed to create message")
}
for position, attachmentID := range req.AttachmentIDs {
if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, attachmentID, position); err != nil {
return nil, apperr.WrapPostgres(err, "failed to link attachment")
for position, id := range message.AttachmentIDs {
if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil {
return apperr.WrapPostgres(err, "failed to link attachment")
}
}
if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version)
VALUES($1,'POST',$2,$3,$4,$5,clock_timestamp()+interval '24 hours',2)
ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,
message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash[:], message.ID); err != nil {
return nil, apperr.WrapPostgres(err, "failed to save message key")
return nil
}
func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error {
_, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires)
if err != nil {
return apperr.WrapPostgres(err, "failed to save message key")
}
return &message, nil
return nil
}
func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) {
@@ -120,20 +112,10 @@ func scanMessage(row pgx.Row) (*dto.Message, error) {
const messageColumns = `id, chat_id, sender_id, text, created_at,
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)`
func messageHash(req dto.CreateMessage) [32]byte {
// Field order and a non-nil list define the canonical v2 representation.
canonical := struct {
Text string `json:"text"`
AttachmentIDs []uuid.UUID `json:"attachment_ids"`
}{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)}
encoded, _ := json.Marshal(canonical) // strings and UUIDs have no encoding failures.
return sha256.Sum256(encoded)
}
func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) {
// Prevent a concurrent integrity update from invalidating the checked rows
// until the message and its links have committed together.
rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids)
rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids)
if err != nil {
return nil, apperr.WrapPostgres(err, "failed to read attachments")
}
@@ -141,7 +123,7 @@ func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, i
result := make([]dto.MessageAttachment, 0, len(ids))
for rows.Next() {
var a dto.MessageAttachment
if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing); err != nil {
if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil {
return nil, apperr.WrapPostgres(err, "failed to read attachment")
}
result = append(result, a)
@@ -0,0 +1,52 @@
package storage
import (
"context"
"errors"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"time"
)
type OutboxRepo struct{ Pool *pgxpool.Pool }
func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) {
_, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt)
return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e
}
func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) {
defer rows.Close()
for rows.Next() {
var m outbox.AcquiredMessage
if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil {
return
}
out = append(out, m)
}
err = rows.Err()
return
}
func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) {
rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED)
UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String())
if e != nil {
return nil, e
}
return scanOutbox(rows)
}
func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) {
rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until<clock_timestamp() LIMIT 100`)
if e != nil {
return nil, e
}
return scanOutbox(rows)
}
func (r *OutboxRepo) Update(ctx context.Context, m outbox.AcquiredMessage, status outbox.Status, next time.Time, code string) error {
tag, e := r.Pool.Exec(ctx, `UPDATE outbox SET status=$3,next_attempt_at=$4,last_error=NULLIF($5,''),locked_until=NULL,updated_at=clock_timestamp() WHERE id=$1 AND token=$2 AND status='processing'`, m.ID, m.Token, status, next, code)
if e == nil && tag.RowsAffected() != 1 {
return errors.New("outbox lease lost")
}
return e
}
+48
View File
@@ -0,0 +1,48 @@
package config
import (
"fmt"
"os"
"strconv"
"strings"
"time"
)
type Pipeline struct {
Concurrency, Prefetch, MaxAttempts int
Timeout, Lease, Retention time.Duration
BrokerURL string
}
func LoadPipeline() (*Pipeline, error) {
c := &Pipeline{Concurrency: 2, Prefetch: 2, MaxAttempts: 5, Timeout: 60 * time.Second, Lease: 90 * time.Second, Retention: 24 * time.Hour}
for k, p := range map[string]*int{"WORKER_CONCURRENCY": &c.Concurrency, "WORKER_PREFETCH": &c.Prefetch, "JOB_MAX_ATTEMPTS": &c.MaxAttempts} {
if v := os.Getenv(k); v != "" {
n, e := strconv.Atoi(v)
if e != nil || n < 1 || n > 64 {
return nil, fmt.Errorf("invalid %s", k)
}
*p = n
}
}
for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} {
if v := os.Getenv(k); v != "" {
d, e := time.ParseDuration(v)
if e != nil || d <= 0 {
return nil, fmt.Errorf("invalid %s", k)
}
*p = d
}
}
if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency {
return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency")
}
if p := os.Getenv("BROKER_URL_FILE"); p != "" {
b, e := os.ReadFile(p)
if e != nil {
return nil, fmt.Errorf("read broker secret: %w", e)
}
c.BrokerURL = strings.TrimSpace(string(b))
}
return c, nil
}
+47
View File
@@ -1 +1,48 @@
package dto
import (
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
"github.com/google/uuid"
"io"
"time"
)
type AttachmentRecord struct {
Public contracts.Attachment
Key string
StagingKey string
PipelineVersion int
VariantKeys map[string]string
Missing bool
}
type UploadAttachment struct {
UserID, ChatID uuid.UUID
Filename, ContentType string
Body io.Reader
}
type AttachmentContent struct {
Body io.ReadCloser
ContentType string
Size int64
}
type JobMessage struct {
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
SchemaVersion int `json:"schema_version"`
JobID uuid.UUID `json:"job_id"`
Type string `json:"type"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
}
type AttachmentJob struct {
Message JobMessage
State string
Token uuid.UUID
LeaseUntil *time.Time
NextAttemptAt time.Time
AttemptLimit int
ErrorCode *string
UpdatedAt time.Time
}
+7
View File
@@ -35,4 +35,11 @@ type MessageAttachment struct {
ID uuid.UUID
ChatID uuid.UUID
Missing bool
Status string
}
type MessageReplay struct {
Hash []byte
HashVersion int
MessageID uuid.UUID
}
+11 -5
View File
@@ -22,7 +22,10 @@ const (
// Defines values for AttachmentStatus.
const (
Ready AttachmentStatus = "ready"
Failed AttachmentStatus = "failed"
Processing AttachmentStatus = "processing"
Queued AttachmentStatus = "queued"
Ready AttachmentStatus = "ready"
)
// Defines values for ErrorErrorCode.
@@ -52,14 +55,17 @@ const (
// Defines values for VariantName.
const (
Crop VariantName = "crop"
Optimized VariantName = "optimized"
Thumbnail VariantName = "thumbnail"
VariantNameCrop VariantName = "crop"
VariantNameOptimized VariantName = "optimized"
VariantNameThumbnail VariantName = "thumbnail"
)
// Defines values for DownloadAttachmentParamsVariant.
const (
Original DownloadAttachmentParamsVariant = "original"
DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop"
DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized"
DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original"
DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail"
)
// Attachment defines model for Attachment.
+64
View File
@@ -0,0 +1,64 @@
// Package fault implements explicit, file-controlled barriers for the isolated lab.
package fault
import (
"context"
"errors"
"github.com/google/uuid"
"os"
"path/filepath"
"strings"
"time"
)
func Hit(ctx context.Context, point string, id uuid.UUID) {
if os.Getenv("LAB5_FAULTS") != "1" {
return
}
p := filepath.Join("/faults", point+"."+id.String())
b, e := os.ReadFile(p)
if e != nil {
p = filepath.Join("/faults", point+".all")
b, e = os.ReadFile(p)
if e != nil {
return
}
}
hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
if e != nil {
return
}
_, _ = hit.WriteString(id.String() + "\n")
_ = hit.Close()
switch strings.TrimSpace(string(b)) {
case "crash":
_ = os.Remove(p)
os.Exit(86)
case "pause":
for {
if _, e = os.Stat(p); os.IsNotExist(e) {
return
}
select {
case <-ctx.Done():
return
case <-time.After(50 * time.Millisecond):
}
}
}
}
func Error(point string, id uuid.UUID) error {
if os.Getenv("LAB5_FAULTS") != "1" {
return nil
}
b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String()))
if e != nil {
return nil
}
v := strings.TrimSpace(string(b))
if v == "temporary" || v == "permanent" {
return errors.New(v)
}
return nil
}
@@ -0,0 +1,83 @@
package imageprocessor
import (
"context"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"os"
"os/exec"
"path/filepath"
"time"
)
func run(ctx context.Context, args ...string) error {
cmd := exec.CommandContext(ctx, "magick", args...)
cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60")
cmd.WaitDelay = time.Second
if err := cmd.Run(); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("image conversion failed: %w", err)
}
return nil
}
func Validate(ctx context.Context, path string) error {
return run(ctx, "-regard-warnings", path, "null:")
}
type Result struct {
Name, Path, MIME string
Width, Height int
}
func fit(w, h, limit int) (int, int) {
if w <= limit && h <= limit {
return w, h
}
if w >= h {
return limit, max(1, h*limit/w)
}
return max(1, w*limit/h), limit
}
func Process(ctx context.Context, source, dir string, version int) ([]Result, error) {
normalized := filepath.Join(dir, "normalized.png")
if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil {
return nil, e
}
f, e := os.Open(normalized)
if e != nil {
return nil, e
}
cfg, _, e := image.DecodeConfig(f)
f.Close()
if e != nil {
return nil, e
}
tw, th := fit(cfg.Width, cfg.Height, 256)
ow, oh := fit(cfg.Width, cfg.Height, 1280)
out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}}
for _, v := range out {
args := []string{normalized}
if v.Name == "crop" {
side := min(cfg.Width, cfg.Height)
args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage")
}
args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality")
quality := "85"
if v.Name == "optimized" && version == 1 {
quality = "80"
}
coder := "jpeg:"
if v.Name == "optimized" {
coder = "webp:"
}
args = append(args, quality, coder+v.Path)
if e = run(ctx, args...); e != nil {
return nil, e
}
}
return out, nil
}
+8
View File
@@ -67,3 +67,11 @@ func Missing(err error) bool {
var api smithy.APIError
return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404")
}
func (s *Store) List(ctx context.Context, token string) (*s3.ListObjectsV2Output, error) {
q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)}
if token != "" {
q.ContinuationToken = &token
}
return s.Client.ListObjectsV2(ctx, q)
}
@@ -0,0 +1,51 @@
package attachment
import (
"context"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"io"
"time"
)
type TransactionManager interface {
RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error
}
type Outbox interface {
CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error)
}
type ObjectStore interface {
Put(context.Context, string, string, io.Reader, int64) error
Get(context.Context, string) (*s3.GetObjectOutput, error)
Head(context.Context, string) (*s3.HeadObjectOutput, error)
Delete(context.Context, string) error
Ping(context.Context) error
List(context.Context, string) (*s3.ListObjectsV2Output, error)
}
type Repository interface {
SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error
Ping(context.Context) error
Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error)
GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error)
InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error
UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error
JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error)
InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error
SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error
ExpiredJobs(context.Context) ([]uuid.UUID, error)
PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error)
FailedOutbox(context.Context) ([]uuid.UUID, error)
ListJobs(context.Context) ([]dto.AttachmentJob, error)
RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error)
AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error
Lock(context.Context, string) (*pgxpool.Conn, func(), error)
Exists(context.Context, *pgxpool.Conn, string) (bool, error)
Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error)
MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error)
CheckConn(context.Context, *pgxpool.Conn) error
}
@@ -0,0 +1,486 @@
package attachment
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
"gitea.privcloud.tech/mtusi2026/internal/pkg/fault"
"gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"io"
"os"
"path/filepath"
"strings"
"time"
)
var ErrBusy = errors.New("job lease held")
func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) }
func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) }
func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
now := time.Now().UTC()
j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now}
if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil {
return e
}
return s.enqueue(ctx, tx, j)
}
func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
b, e := json.Marshal(j.Message)
if e != nil {
return e
}
_, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts})
return e
}
func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) {
a, e := s.Get(ctx, user, id)
if e != nil {
return nil, e
}
if a.Public.Status != "ready" {
return nil, apperr.New(apperr.CodeConflict, "attachment not ready")
}
key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes)
if variant != "original" {
key = a.VariantKeys[variant]
if key == "" {
return nil, apperr.New(apperr.CodeNotFound, "variant not found")
}
for _, v := range a.Public.Variants {
if string(v.Name) == variant {
mime = string(v.ContentType)
size = int64(v.SizeBytes)
}
}
}
obj, e := s.Store.Get(ctx, key)
if e != nil {
return nil, Unavailable()
}
return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil
}
func (s *Service) Process(ctx context.Context, m dto.JobMessage) error {
var j *dto.AttachmentJob
var a *dto.AttachmentRecord
claimed := false
err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
var e error
j, e = s.Repo.JobTx(ctx, tx, m.JobID)
if errors.Is(e, pgx.ErrNoRows) {
return nil
}
if e != nil {
return e
}
// Broker data is a notification. Trusted metadata and attempt live in PostgreSQL.
if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt {
return nil
}
a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID)
if e != nil {
return e
}
if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" {
return nil
}
if j.State == "processing" {
return ErrBusy
}
if time.Now().Before(j.NextAttemptAt) {
return ErrBusy
}
j.State = "processing"
j.Token = uuid.New()
lease := time.Now().Add(s.Jobs.Lease)
j.LeaseUntil = &lease
j.UpdatedAt = time.Now().UTC()
a.Public.Status = "processing"
a.Public.UpdatedAt = j.UpdatedAt
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
return e
}
if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
return e
}
claimed = true
return nil
})
if err != nil || !claimed {
return err
}
s.Fault(ctx, "after_claim", m.JobID)
task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout)
defer cancel()
result, e := s.effects(task, j, a)
if e != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return s.finish(ctx, j, nil, e)
}
s.Fault(ctx, "after_objects", m.JobID)
if err = s.finish(ctx, j, result, nil); err != nil {
return err
}
s.Fault(ctx, "after_result_commit", m.JobID)
return nil
}
type processed struct {
Key string
Variants []contracts.Variant
Keys map[string]string
}
type permanentError struct{ code string }
func (e permanentError) Error() string { return e.code }
func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) {
dir, e := os.MkdirTemp("", "pipeline-*")
if e != nil {
return nil, e
}
defer os.RemoveAll(dir)
key := a.StagingKey
if key == "" {
key = a.Key
}
obj, e := s.Store.Get(ctx, key)
if e != nil {
if objectstore.Missing(e) {
return nil, permanentError{"source_missing"}
}
return nil, e
}
defer obj.Body.Close()
path := filepath.Join(dir, "source")
f, e := os.Create(path)
if e != nil {
return nil, e
}
hash := sha256.New()
n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1))
f.Close()
if e != nil {
return nil, e
}
if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 {
return nil, permanentError{"source_corrupt"}
}
f, e = os.Open(path)
if e != nil {
return nil, e
}
e = s.Validate(ctx, f, string(a.Public.ContentType))
f.Close()
if e != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, permanentError{"invalid_image"}
}
s.Fault(ctx, "before_objects", j.Message.JobID)
if e := fault.Error("before_objects", j.Message.JobID); e != nil {
if e.Error() == "permanent" {
return nil, permanentError{"invalid_image"}
}
return nil, e
}
converted, e := imageprocessor.Process(ctx, path, dir, j.Message.PipelineVersion)
if e != nil {
return nil, e
}
prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion)
result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}}
f, e = os.Open(path)
if e != nil {
return nil, e
}
e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n)
f.Close()
if e != nil {
return nil, e
}
s.Fault(ctx, "after_original_put", j.Message.JobID)
for _, v := range converted {
b, e := os.ReadFile(v.Path)
if e != nil {
return nil, e
}
sum := fmt.Sprintf("%x", sha256.Sum256(b))
key := prefix + v.Name + "/" + sum
if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil {
return nil, e
}
result.Keys[v.Name] = key
result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum})
}
return result, nil
}
func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error {
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID)
if e != nil {
return e
}
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
if e != nil {
return e
}
if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion {
return nil
}
if cause != nil {
return s.retry(ctx, tx, j, a, cause)
}
if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil {
return e
}
for _, v := range result.Variants {
if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil {
return e
}
}
j.State = "ready"
j.ErrorCode = nil
j.LeaseUntil = nil
j.UpdatedAt = time.Now().UTC()
a.Missing = false
a.Key = result.Key
a.VariantKeys = result.Keys
a.Public.Variants = result.Variants
a.Public.Status = "ready"
a.Public.ErrorCode = nil
a.Public.UpdatedAt = j.UpdatedAt
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
return e
}
return s.Repo.UpdateTx(ctx, tx, a)
})
}
func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error {
code := "processing_failed"
var permanent permanentError
terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit
if permanent.code != "" {
code = permanent.code
}
if terminal && permanent.code == "" {
code = "retry_exhausted"
}
j.LeaseUntil = nil
j.Token = uuid.New()
j.UpdatedAt = time.Now().UTC()
j.ErrorCode = &code
if terminal {
j.State = "failed"
a.Public.Status = "failed"
a.Public.ErrorCode = &code
} else {
j.State = "queued"
j.Message.Attempt++
j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1))
a.Public.Status = "queued"
a.Public.ErrorCode = nil
if e := s.enqueue(ctx, tx, j); e != nil {
return e
}
}
a.Public.UpdatedAt = j.UpdatedAt
if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil {
return e
}
return s.Repo.UpdateTx(ctx, tx, a)
}
func (s *Service) Recover(ctx context.Context) error {
ids, e := s.Repo.ExpiredJobs(ctx)
if e != nil {
return e
}
for _, id := range ids {
e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
j, e := s.Repo.JobTx(ctx, tx, id)
if e != nil {
return e
}
if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) {
return nil
}
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
if e != nil {
return e
}
if a.PipelineVersion != j.Message.PipelineVersion {
return nil
}
return s.retry(ctx, tx, j, a, errors.New("lease expired"))
})
if e != nil {
return e
}
}
dead, e := s.Repo.FailedOutbox(ctx)
if e != nil {
return e
}
for _, id := range dead {
if e = s.PublishFailed(ctx, id); e != nil {
return e
}
}
return nil
}
func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error {
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
j, e := s.Repo.JobTx(ctx, tx, id)
if e != nil {
return e
}
if j.State != "queued" {
return nil
}
dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt)
if e != nil {
return e
}
if !dead {
return nil
}
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
if e != nil {
return e
}
if a.PipelineVersion != j.Message.PipelineVersion {
return nil
}
code := "publish_failed"
j.State = "failed"
j.ErrorCode = &code
j.UpdatedAt = time.Now().UTC()
a.Public.Status = "failed"
a.Public.ErrorCode = &code
a.Public.UpdatedAt = j.UpdatedAt
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
return e
}
return s.Repo.UpdateTx(ctx, tx, a)
})
}
func (s *Service) Replay(ctx context.Context, id uuid.UUID) error {
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
j, e := s.Repo.JobTx(ctx, tx, id)
if e != nil {
return e
}
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
if e != nil {
return e
}
if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion {
return apperr.New(apperr.CodeConflict, "only current failed job can be replayed")
}
source := a.StagingKey
if source == "" {
return errors.New("source unavailable; restore original first")
}
if _, e = s.Store.Head(ctx, source); e != nil {
return errors.New("source unavailable")
}
j.State = "queued"
j.Message.Attempt++
j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1
j.NextAttemptAt = time.Now().UTC()
j.UpdatedAt = j.NextAttemptAt
j.Token = uuid.New()
j.ErrorCode = nil
j.LeaseUntil = nil
a.Public.Status = "queued"
a.Public.ErrorCode = nil
a.Public.UpdatedAt = j.UpdatedAt
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
return e
}
if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
return e
}
if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil {
return e
}
return s.enqueue(ctx, tx, j)
})
}
func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error {
if version < 1 || version > 2 {
return errors.New("supported pipeline versions: 1, 2")
}
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
a, e := s.Repo.GetTx(ctx, tx, id)
if e != nil {
return e
}
if version <= a.PipelineVersion || a.Public.Status != "ready" {
return errors.New("requires ready attachment and newer version")
}
a.PipelineVersion = version
a.StagingKey = a.Key
a.Public.Status = "queued"
a.Public.Variants = []contracts.Variant{}
a.VariantKeys = map[string]string{}
a.Public.UpdatedAt = time.Now().UTC()
if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
return e
}
return s.newJob(ctx, tx, a)
})
}
func (s *Service) Cleanup(ctx context.Context) error {
records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention))
if e != nil {
return e
}
for _, record := range records {
// Serialize deletion with operator Restore and the initial upload handoff.
_, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey)
if lockErr != nil {
return lockErr
}
e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
a, e := s.Repo.GetTx(ctx, tx, record.Public.Id)
if e != nil {
return e
}
if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" {
return nil
}
if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) {
return nil
}
if !strings.HasPrefix(a.StagingKey, "results/") {
if e = s.Store.Delete(ctx, a.StagingKey); e != nil {
return e
}
}
a.StagingKey = ""
return s.Repo.UpdateTx(ctx, tx, a)
})
unlock()
if e != nil {
return e
}
}
return nil
}
func managedPipelineKey(key string) bool {
return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/")
}
@@ -10,8 +10,9 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
)
@@ -38,17 +39,17 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
result.Apply = apply
result.GraceSeconds = s.Config.Grace.Seconds()
defer func() { result.DurationMS = time.Since(start).Milliseconds() }()
if err = s.Repo.Pool.Ping(ctx); err != nil {
if err = s.Repo.Ping(ctx); err != nil {
return
}
result.S3Requests++
if err = s.Store.Ping(ctx); err != nil {
return
}
pager := s3.NewListObjectsV2Paginator(s.Store.Client, &s3.ListObjectsV2Input{Bucket: &s.Store.Bucket, Prefix: aws.String(Prefix), MaxKeys: aws.Int32(100)})
for pager.HasMorePages() {
token := ""
for {
result.S3Requests++
page, e := pager.NextPage(ctx)
page, e := s.Store.List(ctx, token)
if e != nil {
return result, e
}
@@ -92,7 +93,7 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
return nil
}
result.DBChecks++
if _, e = conn.Exec(ctx, "SELECT 1"); e != nil {
if e = s.Repo.CheckConn(ctx, conn); e != nil {
return e
}
result.S3Requests++
@@ -106,7 +107,12 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
return result, e
}
}
if !aws.ToBool(page.IsTruncated) {
break
}
token = aws.ToString(page.NextContinuationToken)
}
after := uuid.Nil
for {
result.DBChecks++
@@ -120,6 +126,9 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
for _, a := range page {
result.Metadata++
after = a.Public.Id
if a.Public.Status != "ready" {
continue
}
result.LockAttempts++
conn, unlock, e := s.Repo.Lock(ctx, a.Key)
if e != nil {
@@ -201,5 +210,19 @@ func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error
return fmt.Errorf("stored original verification failed")
}
_, e = s.Repo.MarkMissing(ctx, conn, id, false)
return e
if e != nil {
return e
}
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
current, e := s.Repo.GetTx(ctx, tx, id)
if e != nil {
return e
}
if current.Public.Status == "failed" {
current.StagingKey = current.Key
current.Public.UpdatedAt = time.Now().UTC()
return s.Repo.UpdateTx(ctx, tx, current)
}
return nil
})
}
+38 -35
View File
@@ -8,8 +8,7 @@ import (
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net/http"
"os"
"strings"
"time"
@@ -18,28 +17,31 @@ import (
"github.com/google/uuid"
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"github.com/jackc/pgx/v5"
)
const Prefix = "originals/"
const StagingPrefix = "staging/"
type Membership interface {
GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error)
}
type Service struct {
Repo *storage.AttachmentsRepo
Store *objectstore.Store
Repo Repository
Store ObjectStore
Config *config.Attachment
chats Membership
tm TransactionManager
outbox Outbox
Jobs *config.Pipeline
uploads, decoders chan struct{}
}
func New(repo *storage.AttachmentsRepo, store *objectstore.Store, c *config.Attachment, chats Membership) *Service {
func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service {
return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)}
}
func ValidFilename(s string) bool {
@@ -53,21 +55,15 @@ func ValidFilename(s string) bool {
}
return true
}
func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid.UUID, filename string) (*contracts.Attachment, error) {
ctx := r.Context()
if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: chat}); e != nil {
func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) {
s.tm = tm
s.outbox = outbox
s.Jobs = jobs
}
func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (*contracts.Attachment, error) {
if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil {
return nil, e
}
if !ValidFilename(filename) {
return nil, apperr.New(apperr.CodeBadRequest, "invalid filename")
}
mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
if e != nil || (mt != "image/png" && mt != "image/jpeg") {
return nil, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")
}
if r.ContentLength > s.Config.MaxBytes {
return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
}
select {
case s.uploads <- struct{}{}:
defer func() { <-s.uploads }()
@@ -80,23 +76,22 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid
}
defer os.Remove(f.Name())
defer f.Close()
r.Body = http.MaxBytesReader(w, r.Body, s.Config.MaxBytes)
hash := sha256.New()
size, e := io.Copy(io.MultiWriter(f, hash), r.Body)
size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1))
if e != nil {
if _, ok := e.(*http.MaxBytesError); ok {
return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
}
return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload")
}
if e = s.Validate(ctx, f, mt); e != nil {
if size > s.Config.MaxBytes {
return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
}
if e = s.Validate(ctx, f, req.ContentType); e != nil {
return nil, e
}
now := time.Now().UTC().Truncate(time.Microsecond)
id := uuid.New()
key := Prefix + id.String()
a := &storage.AttachmentRecord{Key: key, Public: contracts.Attachment{Id: id, ChatId: chat, OwnerId: user, Filename: filename, ContentType: contracts.AttachmentContentType(mt), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "ready", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}}
conn, unlock, e := s.Repo.Lock(ctx, key)
key := StagingPrefix + id.String()
a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}}
_, unlock, e := s.Repo.Lock(ctx, key)
if e != nil {
return nil, e
}
@@ -104,14 +99,19 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid
if _, e = f.Seek(0, 0); e != nil {
return nil, e
}
if e = s.Store.Put(ctx, key, mt, f, size); e != nil {
if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil {
return nil, Unavailable()
}
// Never blindly compensate a failed/ambiguous COMMIT with DELETE: a committed
// row may already reference this key. Reconciliation decides after grace.
if e = s.Repo.Save(ctx, conn, a); e != nil {
s.Fault(ctx, "after_staging", id)
if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
if e := s.Repo.InsertTx(ctx, tx, a); e != nil {
return e
}
return s.newJob(ctx, tx, a)
}); e != nil {
return nil, e
}
s.Fault(ctx, "after_upload_commit", id)
return &a.Public, nil
}
func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error {
@@ -134,12 +134,12 @@ func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error {
if _, e = f.Seek(0, 0); e != nil {
return e
}
if _, _, e = image.Decode(f); e != nil {
if e = validateImage(ctx, f.Name()); e != nil {
return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image")
}
return ctx.Err()
}
func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.AttachmentRecord, error) {
func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) {
a, e := s.Repo.Get(ctx, id)
if e != nil {
return nil, e
@@ -154,6 +154,9 @@ func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.Attachm
}
func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") }
func ManagedKey(key string) bool {
if managedPipelineKey(key) {
return true
}
if !strings.HasPrefix(key, Prefix) {
return false
}
@@ -0,0 +1,54 @@
package chat
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"gitea.privcloud.tech/mtusi2026/internal/dto"
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"time"
)
// CreateIdempotentTx performs the message use case inside an existing transaction.
func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) {
scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey
if err := repo.LockKeyTx(ctx, tx, scope); err != nil {
return nil, err
}
canonical := struct {
Text string `json:"text"`
AttachmentIDs []uuid.UUID `json:"attachment_ids"`
}{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)}
encoded, err := json.Marshal(canonical)
if err != nil {
return nil, err
}
hash := sha256.Sum256(encoded)
replay, err := repo.ReplayTx(ctx, tx, req)
if err != nil {
return nil, err
}
if replay != nil {
if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 {
hash = sha256.Sum256([]byte(req.Text))
} else if replay.HashVersion != 2 {
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
}
if !bytes.Equal(hash[:], replay.Hash) {
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
}
return repo.MessageTx(ctx, tx, replay.MessageID)
}
now := time.Now().UTC().Truncate(time.Microsecond)
message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now}
if err = repo.InsertMessageTx(ctx, tx, message); err != nil {
return nil, err
}
if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil {
return nil, err
}
return &message, nil
}
+6 -1
View File
@@ -2,6 +2,7 @@ package chat
import (
"context"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -17,7 +18,11 @@ type chatsRepo interface {
type messagesRepo interface {
AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error)
CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error)
LockKeyTx(context.Context, pgx.Tx, string) error
ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error)
MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error)
InsertMessageTx(context.Context, pgx.Tx, dto.Message) error
SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error
GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error)
ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error)
}
+4 -1
View File
@@ -190,6 +190,9 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M
if attachment.ChatID != req.ChatID {
return apperr.New(apperr.CodeNotFound, "attachment not found")
}
if attachment.Status == "failed" {
return apperr.New(apperr.CodeConflict, "failed attachment")
}
if attachment.Missing {
return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable")
}
@@ -197,7 +200,7 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M
}
var err error
message, err = c.messagesRepo.CreateIdempotentTx(ctx, tx, req)
message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req)
return err
})
return message, err
+12
View File
@@ -0,0 +1,12 @@
package outbox
import (
"context"
"github.com/jackc/pgx/v5"
)
type API interface {
CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error)
Create(context.Context, CreateMessageReq) (CreateMessageRes, error)
}
+115
View File
@@ -0,0 +1,115 @@
package outbox
import (
"time"
"github.com/google/uuid"
)
type CreateMessageReq struct {
ID uuid.UUID
CreatedAt time.Time
Transport Transport
AggregateType string
AggregateID uuid.UUID
MessageType string
IdempotencyKey *string
SchemaVersion int
CausationID *uuid.UUID
Metadata map[string]string
Payload []byte
MaxAttempts int
Status Status
}
type CreateMessageRes struct {
ID uuid.UUID
CreatedAt time.Time
}
type MarkPublishedMessageReq struct {
ID uuid.UUID
UpdatedAt time.Time
PublishedAt time.Time
}
type MarkFailedMessageReq struct {
ID uuid.UUID
UpdatedAt time.Time
NextAttemptAt time.Time
LastError string
}
type MarkDeadLetterMessageReq struct {
ID uuid.UUID
UpdatedAt time.Time
DeadLetterAt time.Time
LastError string
}
type AcquireMessagesForProcessingReq struct {
Limit uint
LockDuration time.Duration
}
type AcquiredMessage struct {
Token uuid.UUID
ID uuid.UUID
Transport Transport
AggregateType string
AggregateID uuid.UUID
MessageType string
SchemaVersion int
CorrelationID uuid.UUID
CausationID *uuid.UUID
Status Status
Metadata map[string]string
Payload []byte
Attempts int
MaxAttempts int
LastError *string
LockedAt *time.Time
LockedUntil *time.Time
NextAttemptAt *time.Time
PublishedAt *time.Time
DeadLetterAt *time.Time
CreatedAt time.Time
UpdatedAt *time.Time
}
type RecoverStuckMessagesAsFailedReq struct {
UpdatedAt time.Time
NextAttemptAt time.Time
LastError string
Limit uint
}
type RecoverStuckMessagesAsDeadLetterReq struct {
UpdatedAt time.Time
DeadLetterAt time.Time
LastError string
Limit uint
}
type DeliveryMessage struct {
ID uuid.UUID
Transport Transport
AggregateType string
AggregateID uuid.UUID
MessageType string
SchemaVersion int
CorrelationID uuid.UUID
CausationID *uuid.UUID
Payload []byte
Metadata map[string]string
}
type ProcessOnceResult struct {
RecoveredAsFailed int64
RecoveredAsDeadLetter int64
Acquired int
Published int
Failed int
DeadLettered int
}
+16
View File
@@ -0,0 +1,16 @@
package outbox
const (
StatusCreated Status = "created"
StatusPending Status = "pending"
StatusProcessing Status = "processing"
StatusPublished Status = "published"
StatusFailed Status = "failed"
StatusDeadLetter Status = "dead_letter"
)
type Status string
func (s Status) String() string {
return string(s)
}
+114
View File
@@ -0,0 +1,114 @@
// Adapted from mamont-optics/application/internal/service/outbox.
// Retains transactional creation, batch acquisition, bounded retries and lease recovery.
package outbox
import (
"context"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"math/rand/v2"
"time"
)
type Repository interface {
CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error)
AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error)
Expired(context.Context) ([]AcquiredMessage, error)
Update(context.Context, AcquiredMessage, Status, time.Time, string) error
}
type TransportPublisher interface {
Publish(context.Context, DeliveryMessage) error
}
type TransactionManager interface {
RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error
}
type Svc struct {
repo Repository
tm TransactionManager
publisher TransportPublisher
OnDead func(context.Context, uuid.UUID) error
}
func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc {
return &Svc{repo: repo, tm: tm, publisher: publisher}
}
func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) {
if r.ID == uuid.Nil {
r.ID = uuid.New()
}
if r.CreatedAt.IsZero() {
r.CreatedAt = time.Now().UTC()
}
if r.Status == "" {
r.Status = StatusPending
}
if r.MaxAttempts == 0 {
r.MaxAttempts = 5
}
return s.repo.CreateTx(ctx, tx, r)
}
func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) {
err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e })
return
}
func Backoff(attempt int) time.Duration {
n := min(max(attempt-1, 0), 5)
cap := min(time.Second*time.Duration(1<<n), 30*time.Second)
return cap/2 + time.Duration(rand.Int64N(int64(cap/2)+1))
}
func (s *Svc) ProcessOnce(ctx context.Context) (result ProcessOnceResult, err error) {
expired, e := s.repo.Expired(ctx)
if e != nil {
return result, e
}
for _, m := range expired {
if e = s.fail(ctx, m, "publish_lease_expired"); e != nil {
return result, e
}
}
messages, e := s.repo.AcquireManyForProcessing(ctx, AcquireMessagesForProcessingReq{Limit: 1, LockDuration: 30 * time.Second})
if e != nil {
return result, e
}
for _, m := range messages {
result.Acquired++
// Publish payload is the job contract, never the outbox envelope.
if s.publisher == nil {
return result, errors.New("publisher unavailable")
}
e = s.publisher.Publish(ctx, DeliveryMessage{ID: m.ID, AggregateID: m.AggregateID, Payload: m.Payload})
if e != nil {
if e = s.fail(ctx, m, "publish_failed"); e != nil {
return result, e
}
result.Failed++
continue
}
if e = s.repo.Update(ctx, m, StatusPublished, time.Now().UTC(), ""); e != nil {
return result, e
}
result.Published++
}
return result, nil
}
func (s *Svc) fail(ctx context.Context, m AcquiredMessage, code string) error {
status := StatusFailed
if m.Attempts >= m.MaxAttempts {
status = StatusDeadLetter
}
if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil {
return err
}
if status == StatusDeadLetter && s.OnDead != nil {
var j struct {
JobID uuid.UUID `json:"job_id"`
}
if err := json.Unmarshal(m.Payload, &j); err != nil {
return err
}
return s.OnDead(ctx, j.JobID)
}
return nil
}
@@ -0,0 +1,7 @@
package outbox
type Transport string
const TransportAMQP Transport = "amqp"
func (t Transport) String() string { return string(t) }
+41
View File
@@ -0,0 +1,41 @@
ALTER TABLE attachments
ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')),
ADD COLUMN error_code text,
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
ADD COLUMN staging_key text NOT NULL DEFAULT '',
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}';
UPDATE attachments SET staging_key=object_key;
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0),
attempt_limit integer NOT NULL, state text NOT NULL,
token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL,
error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL,
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until);
CREATE TABLE outbox (
id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
payload jsonb NOT NULL, status text NOT NULL,
attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL,
token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL,
last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL
);
CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at);
CREATE TABLE job_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Existing lab4 originals become durable sources for the first asynchronous run.
INSERT INTO attachment_jobs
SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments;
INSERT INTO outbox
SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at),
'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs;
---- create above / drop below ----
DROP TABLE job_audit;
DROP TABLE outbox;
DROP TABLE attachment_jobs;
ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys;
@@ -0,0 +1,8 @@
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')),
object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL,
PRIMARY KEY(attachment_id,pipeline_version,variant)
);
---- create above / drop below ----
DROP TABLE attachment_variants;
+27 -5
View File
@@ -69,11 +69,12 @@ func TestAttachmentContractAndErrors(t *testing.T) {
Sha string `json:"sha256"`
Size int `json:"size_bytes"`
}
decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 201, &att)
decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att)
hash := sha256.Sum256(data)
if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) {
t.Fatal("original metadata mismatch")
}
waitAttachmentReady(t, api, alice, att.ID)
content := "/api/v1/attachments/" + att.ID + "/content"
got := api.request(t, "GET", content, bob, "")
if got.status != 200 || !bytes.Equal(got.body, data) {
@@ -115,7 +116,7 @@ func TestAttachmentContractAndErrors(t *testing.T) {
boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk))
boundary = append(boundary, data[len(data)-12:]...)
var boundaryAtt map[string]any
decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 201, &boundaryAtt)
decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt)
oversized := make([]byte, 10485761)
copy(oversized, data)
@@ -127,7 +128,7 @@ func TestAttachmentContractAndErrors(t *testing.T) {
t.Fatal(e)
}
var jpegAtt map[string]any
decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 201, &jpegAtt)
decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt)
imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}`
var message map[string]any
key := uuid.NewString()
@@ -145,7 +146,9 @@ func TestAttachmentContractAndErrors(t *testing.T) {
for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} {
assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request")
}
assertError(t, api.request(t, "GET", content+"?variant=thumbnail", bob, ""), 400, "invalid_request")
if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 {
t.Fatal("thumbnail unavailable")
}
}
func TestConcurrentAttachmentMessage(t *testing.T) {
api := newAPI(t)
@@ -154,7 +157,7 @@ func TestConcurrentAttachmentMessage(t *testing.T) {
var att struct {
ID string `json:"id"`
}
decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 201, &att)
decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att)
key := uuid.NewString()
body := `{"text":"","attachment_ids":["` + att.ID + `"]}`
type result struct {
@@ -202,3 +205,22 @@ func TestConcurrentAttachmentMessage(t *testing.T) {
t.Fatal("duplicate history")
}
}
func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) {
t.Helper()
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
var att struct {
Status string `json:"status"`
}
decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att)
if att.Status == "ready" {
return
}
if att.Status == "failed" {
t.Fatal("attachment failed")
}
time.Sleep(100 * time.Millisecond)
}
t.Fatal("attachment readiness deadline")
}
+14
View File
@@ -0,0 +1,14 @@
# Проверки решения lab5
Публичные `tests/` не изменены. Собственные проверки используют их HTTP-клиент и создают только синтетические данные в локальном стенде.
- `make test-images`: реальные форматы и размеры вариантов, floor/no-upscale, EXIF orientation, белый фон alpha, центральный crop, hashes, ACL и отрицательные входы.
- `make test-staging`: защита queued-source, удаление staging после ready, terminal source_missing, 24h retention и восстановление с явным replay. Для retention меняется только возраст одной синтетической записи на 25h; реальный день не ожидается и конфигурация retention не сокращается.
- `make test-broker-persistence`: заменяет контейнер брокера при остановленных worker и dispatcher; проверяет, что уже опубликованное сообщение осталось в очереди с тем же volume.
- `make test-pipeline`: 13 сценариев × 3 повтора, включая crash, дубли, lease recovery, reorder, устаревший worker и версию pipeline. На время выполнения подключает `compose.faults.yaml`, перезапускает процессы и stores с сохранением volumes. В конце восстанавливает обычный Compose без fault injection.
Перед запуском нужен `make up`. Проверки, которые останавливают/перезапускают процессы (`test-staging`, `test-pipeline`, persistence/failover), запускаются последовательно. Результаты — `evidence/*.json`; секреты и токены туда не записываются. Директивы fault injection лежат в игнорируемой `.lab5-faults/`.
`jobs redeliver --id JOB_UUID` — только инструмент экспериментального профиля с `LAB5_FAULTS=1`, публикующий дополнительную копию сохранённой job. Обычный оператор использует `jobs replay` для failed или `jobs reprocess` для новой версии.
Исследование и ограничения описаны в [REPORT.md](../REPORT.md).
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Prove a confirmed queued message survives broker container replacement, not only restart."""
from pathlib import Path
import json
import subprocess
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check
def compose(*args):
r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150)
check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip()
def queued():
r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30)
if r.returncode:return 0
out=r.stdout
for line in out.splitlines():
fields=line.split()
if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1])
return 0
def main():
c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap()
try:
compose('stop','worker');att,data=c.attachment(alice,chat)
deadline=time.monotonic()+20
while queued()<1 and time.monotonic()<deadline:time.sleep(.2)
check(queued()>=1,'message not durably queued before replacement')
published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id'])
check(published=='t','outbox not marked published before replacement')
compose('stop','dispatcher')
old=compose('ps','-q','broker')
compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker')
new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced')
deadline=time.monotonic()+30
while queued()<1 and time.monotonic()<deadline:time.sleep(.3)
check(queued()>=1,'confirmed queue message lost after replacement')
compose('start','worker');c.wait_ready(bob,att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement')
result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True}
(ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n')
print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity')
finally:compose('start','worker','dispatcher')
if __name__=='__main__':main()
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""Image contract checks using synthetic fixtures and real decoding in ImageMagick."""
import hashlib
import json
from pathlib import Path
import struct
import subprocess
import sys
import urllib.parse
import zlib
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check
from image_probe import jpeg_size
def magick(args,data=None):
r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30)
check(r.returncode==0,'fixture decoder failed')
return r.stdout
def png(w,h,rgba):
def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b))
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'')
def main():
c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[]
jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255)))
# EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise.
tiff=b'II'+struct.pack('<HI',42,8)+struct.pack('<H',1)+struct.pack('<HHI',0x112,3,1)+struct.pack('<H',6)+b'\0\0'+struct.pack('<I',0)
exif=b'Exif\0\0'+tiff;oriented=jpeg[:2]+b'\xff\xe1'+struct.pack('>H',len(exif)+2)+exif+jpeg[2:]
for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]:
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att)
original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed')
for v in ready['variants']:
b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch')
actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split()))
check(actual==(v['width'],v['height']),'actual dimensions differ')
if v['name']=='thumbnail':
check(jpeg_size(b)==dimensions,'thumbnail geometry differs')
if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white')
if v['name']=='crop':
check(actual==(128,128),'crop geometry differs')
if name=='center_crop':
import re
color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b))))
check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered')
check(b'Exif\0\0' not in b,'EXIF retained in derivative')
c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve)
results.append({'case':name,'result':'PASS','thumbnail':dimensions})
for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]:
c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime})
(ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs')
if __name__=='__main__':main()
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env python3
"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens."""
import argparse
import concurrent.futures
import datetime
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
FAULTS=ROOT/'.lab5-faults'
COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')]
RESULTS=[]
SELECTED=set()
def compose(*args, allow_failure=False):
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180)
if p.returncode and not allow_failure:
# Infrastructure logs may contain settings; report only action and exit status.
raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode))
return p
def sql(query):
return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip()
def uid(value):
return str(uuid.UUID(value))
def job(att,version=1):
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version))
def operator(action,id=None,version=None, allow_failure=False):
args=['run','--rm','--no-deps','-T','operator','jobs',action]
if id: args+=['--id',uid(id)]
if version:args+=['--pipeline-version',str(version)]
return compose(*args,allow_failure=allow_failure)
def wait(predicate,timeout=65):
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
v=predicate()
if v:return v
time.sleep(.1)
raise Failure('experiment condition timed out')
def directive(point,id,action):
p=FAULTS/(point+'.'+id)
Path(str(p)+'.hit').unlink(missing_ok=True)
p.write_text(action);p.chmod(0o666)
return p
def hit(p,timeout=65):
wait(lambda:Path(str(p)+'.hit').exists(),timeout)
return Path(str(p)+'.hit').read_text().strip()
def metadata(c,actor,att):
return c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
def state(c,actor,att,value,timeout=65):
return wait(lambda:(a if (a:=metadata(c,actor,att))['status']==value else None),timeout)
def ready(c,actor,att,content):
a=c.wait_ready(actor,att)
check(a['error_code'] is None,'ready has error')
original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']
check(original==content,'original changed')
variants={v['name']:v for v in a['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'variant set differs')
for name,v in variants.items():
r=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
check(hashlib.sha256(r['bytes']).hexdigest()==v['sha256'] and len(r['bytes'])==v['size_bytes'],'variant checksum mismatch')
if name!='optimized':check(jpeg_size(r['bytes'])==(v['width'],v['height']),'JPEG dimensions differ')
else:check(r['bytes'][:4]==b'RIFF' and r['bytes'][8:12]==b'WEBP','optimized is not WebP')
rows=int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))
check(rows in (4,8),'manifest has duplicate/missing logical variants')
return a
def record(name,run,fn):
if SELECTED and name not in SELECTED:return
start=time.monotonic();detail=fn() or {}
r={'scenario':name,'run':run,'result':'PASS','seconds':round(time.monotonic()-start,3),**detail}
RESULTS.append(r);print(json.dumps(r),flush=True)
def main():
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);p.add_argument('--base-url',default='http://localhost:18085');p.add_argument('--output',default='evidence/pipeline.json');p.add_argument('--scenario',action='append',default=[]);args=p.parse_args()
SELECTED.update(args.scenario)
FAULTS.mkdir(exist_ok=True);FAULTS.chmod(0o777)
for f in FAULTS.iterdir():f.unlink()
compose('up','-d','--build','--wait','--wait-timeout','180')
c=Client(args.base_url,timeout=15);alice,bob,eve,chat=c.bootstrap()
def upload_stopped():
compose('stop','worker');att,content=c.attachment(alice,chat);return att,content,job(att)
try:
for run in range(1,args.repeats+1):
def recovery():
att,data,jid=upload_stopped();c.message(alice,chat,'',attachments=[att['id']]);c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
before=sql('SELECT pg_postmaster_start_time()')
compose('restart','api','postgres','redis','s3','broker')
wait(lambda:health(c),90);compose('run','--rm','--no-deps','-T','s3-init');compose('start','worker');ready(c,bob,att,data)
check(sql('SELECT pg_postmaster_start_time()')!=before,'PostgreSQL was not restarted')
return {'attachment_id':att['id'],'job_id':jid,'postgres_restarted':True}
record('worker_down_and_store_restart',run,recovery)
for point in ('before_objects','after_original_put','after_objects','after_result_commit','before_ack'):
def crash(point=point):
att,data,jid=upload_stopped();marker=directive(point,jid,'crash');compose('start','worker');hit(marker);ready(c,bob,att,data)
return {'job_id':jid,'manifest_rows':int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))}
record('crash_'+point,run,crash)
def duplicates():
att,data=c.attachment(alice,chat);first=ready(c,bob,att,data);jid=job(att)
for _ in range(3):operator('redeliver',jid)
time.sleep(.5);check(metadata(c,bob,att)==first,'ready changed after duplicate')
return {'job_id':jid,'copies':3}
record('ready_redelivery',run,duplicates)
def publish_crash():
att,data,jid=upload_stopped();compose('stop','dispatcher');marker=directive('after_publish',jid,'crash')
# Existing publish may have completed before stop. Add a second durable notification.
sql("UPDATE outbox SET status='pending',next_attempt_at=clock_timestamp() WHERE job_id='%s'"%uid(jid))
compose('start','dispatcher');hit(marker);compose('start','worker');ready(c,bob,att,data)
wait(lambda:sql("SELECT count(*) FROM outbox WHERE job_id='%s' AND status='published'"%uid(jid))!='0',45)
return {'job_id':jid}
record('crash_confirm_before_outbox_mark',run,publish_crash)
def version_fence():
att,data,jid=upload_stopped();marker=directive('after_objects',jid,'pause');compose('start','worker');hit(marker)
# Recovery expires the real 8s lease; a newer attempt finishes on the second worker slot.
first=ready(c,bob,att,data);check(int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid)))>=2,'lease not recovered')
operator('reprocess',att['id'],2);second=ready(c,bob,att,data)
check(first['variants']!=second['variants'],'pipeline versions are indistinguishable')
operator('redeliver',jid);marker.unlink();time.sleep(1)
check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2')
check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back')
ready(c,bob,att,data)
return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8}
record('lease_reorder_and_stale_version',run,version_fence)
def retries():
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65)
check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed')
return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6}
record('bounded_retry_and_explicit_replay',run,retries)
def permanent():
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid}
record('permanent_error',run,permanent)
for point in ('after_staging','after_upload_commit'):
def api_crash(point=point):
if point=='after_upload_commit':compose('stop','dispatcher')
marker=directive(point,'all','crash')
try:c.attachment(alice,chat)
except Failure:pass
aid=hit(marker);att={'id':aid}
if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists')
else:
check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached')
compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes())
wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'}
record('api_crash_'+point,run,api_crash)
payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS}
check(bool(RESULTS),'no scenarios selected')
payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab5','--format','{{.Id}}'],text=True).strip()
(ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n')
finally:
for f in FAULTS.iterdir():f.unlink()
compose('stop','worker','dispatcher',allow_failure=True)
# Restore standard configuration, disabling fault injection and short experiment leases.
subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180)
def health(c):
try:c.call('GET','/health/ready',schema='Health');return True
except Failure:return False
if __name__=='__main__':main()
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Staging protection, retirement, retention and explicit restoration on synthetic data."""
import json
from pathlib import Path
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check
BASE=[str(ROOT/'scripts/compose.sh')]
import subprocess
def command(*args,fail=False):
r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120)
if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0])
return r
def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip()
def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail)
def wait(fn):
deadline=time.monotonic()+20
while time.monotonic()<deadline:
if fn():return
time.sleep(.1)
raise AssertionError('storage condition timed out')
def exists(key):return json.loads(op('storage','object-check','--key',key).stdout)['exists']
def main():
c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[]
try:
command('stop','worker')
att,data=c.attachment(alice,chat);key='staging/'+att['id']
check(exists(key),'accepted source missing')
result=json.loads(op('storage','reconcile','--apply').stdout)
check(result['referenced_objects']>0 and exists(key),'reconciliation removed active source')
results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']})
command('start','worker');c.wait_ready(bob,att)
wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='')
check(not exists(key),'ready source staging not retired')
results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']})
command('stop','worker')
failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id'])
op('storage','remove-object','--id',failed['id']);command('start','worker')
def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed'
wait(is_failed)
check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent')
check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source')
op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png')
check(exists(key),'restored source missing')
# Advance only the synthetic record's age; do not wait a real day or change production retention.
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id'])
wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='')
check(not exists(key),'expired failed source not retired')
check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source')
results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25})
op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed)
check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed')
results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid})
(ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n')
print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified')
finally:command('start','worker')
if __name__=='__main__':main()