diff --git a/.dockerignore b/.dockerignore index bc16e55..19e41e7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.lab5-faults +bin diff --git a/.env.example b/.env.example index ecbfeb2..b0567a9 100644 --- a/.env.example +++ b/.env.example @@ -30,9 +30,11 @@ RECONCILE_GRACE_PERIOD=1h # SeaweedFS image and persistent bucket bootstrap are pinned in compose.yaml. # API has four upload slots and one decoder per replica, GOMEMLIMIT=384MiB. -# Planned lab5 async pipeline settings; the ported lab4 application does not use these yet. -BROKER_URL_FILE=/run/secrets/broker_url +# Async lab5 pipeline. Broker credentials are generated by bootstrap. +BROKER_PASSWORD=replace-with-local-broker-password WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +JOB_TIMEOUT=60s +JOB_LEASE=90s +STAGING_RETENTION=24h diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..0ae3427 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,28 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.2' + cache-dependency-path: service/go.sum + - run: make check build vet + - run: make up + - run: make test test-go test-storage test-images test-staging test-broker-persistence + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh + - run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh + - run: make restore + - run: make test-pipeline + - uses: actions/upload-artifact@v4 + if: always() + with: + name: pipeline-evidence + path: evidence/pipeline.json + - run: make down + if: always() diff --git a/.gitignore b/.gitignore index 73a022b..01540a7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ coverage/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. /.ignore/ + +.lab5-faults/ diff --git a/Dockerfile b/Dockerfile index 38980df..a953e59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 +RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0 +COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml + COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger ENV APP_HOST=0.0.0.0 \ diff --git a/Makefile b/Makefile index db8c42a..55b3c45 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen .PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt help: - @echo "up/down: lab5 stand (ported lab4 application); build/vet: Go checks" - @echo "test: lab5 API smoke (requires async implementation); test-go/test-storage: existing Go checks" + @echo "up/down: lab5 async image pipeline; build/vet: Go checks" + @echo "test: lab5 API smoke; test-go/test-storage: Go checks" @echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic" @echo "check/validate: lab5 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks" @echo "reconcile/reconcile-apply: storage consistency" @@ -111,3 +111,18 @@ reconcile: reconcile-apply: ./scripts/compose.sh run --rm -T operator storage reconcile --apply + +.PHONY: test-pipeline +test-pipeline: + $(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json + +.PHONY: test-images test-staging +test-images: + $(PYTHON) student-tests/images.py + +test-staging: + $(PYTHON) student-tests/storage.py + +.PHONY: test-broker-persistence +test-broker-persistence: + $(PYTHON) student-tests/broker_persistence.py diff --git a/REPORT.md b/REPORT.md index e98f75e..489af1e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,173 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не предоставлены владельцем проекта. +- Целевая оценка: **5**, трек 2 — доказанная идемпотентность versioned pipeline. +- Ревизия: базовый commit и состояние рабочего дерева записываются в `evidence/pipeline.json`; решение пока не закоммичено. +- Использованы Go, pgx, AWS SDK, RabbitMQ `amqp091-go`, ImageMagick; ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Compose и для локальных Go-проверок Go 1.27+. Контейнерная сборка использует Go 1.27.2. Образы закреплены версиями/digest в `compose.yaml` и `Dockerfile`; ImageMagick и модули JPEG/WebP — 7.1.2.35-r0. -## Архитектура и решения +```sh +make check build vet +make up +curl --fail http://localhost:18085/health/ready +make test +make test-go +make test-storage +make test-images test-staging test-broker-persistence +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make restore +make test-pipeline +make down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Полная валидация схем: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +Bootstrap сохраняет существующие локальные настройки и генерирует синтетические credentials PostgreSQL, Redis, S3 и RabbitMQ. `.env.lab5` и `secrets/lab5` игнорируются Git. Миграции выполняются отдельным одноразовым процессом, bucket создаётся идемпотентно. Внешний endpoint — только `127.0.0.1:18085`; внутренние хранилища портов на host не публикуют. API-реплик по умолчанию две. + +`make down` сохраняет volumes. **Намеренное удаление учебных данных**: `./scripts/compose.sh down --volumes`; эта команда не входит в обычные проверки и автоматически не выполняется. + +## Архитектура и границы + +```mermaid +flowchart LR + Client --> LB[HAProxy] + LB --> API[API x2] + API --> Staging[Private S3 staging] + API --> DB[PostgreSQL: metadata + jobs + outbox] + DB --> Dispatcher + Dispatcher --> Broker[RabbitMQ durable queue] + Broker --> Worker + Worker --> Results[Private S3 versioned results] + Worker --> DB +``` + +Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Attachment service принимает DTO и reader, проверяет доступ и целостность изображения, управляет транзакцией и состояниями. Репозитории выполняют операции хранения; HTTP, retry-политик и преобразования изображений в SQL/триггерах нет. Канонизация тела сообщения, сравнение идемпотентных запросов и создание Message перенесены в chat service. + +До `202` весь проверенный оригинал уже записан в S3 staging, а metadata, job и outbox — в одной завершённой транзакции PostgreSQL. При неопределённом COMMIT нет слепого компенсирующего DELETE. Ответ содержит сохранённый снимок `queued`, даже если быстрый worker уже начал работу. + +Staging находится в том же S3-сервисе, что конечные объекты: это разделение prefix и жизненных циклов, а не обещание отложить всю сетевую передачу файла. Проверка корректности изображения при upload включает ограниченное декодирование; resize/crop/WebP и публикация конечного оригинала выполняются отдельно в worker. + +API readiness проверяет PostgreSQL и S3. Недоступность broker/worker не блокирует транзакционный приём через outbox. Worker и dispatcher переподключаются после отказа зависимостей. Redis остаётся необязательным кэшем, очередь в него не помещается. + +### Инфраструктура + +| Роль → service | Версия | Проверка | Volume / сеть / credentials | +| --- | --- | --- | --- | +| API → api, ≥2 | собственная сборка Go 1.27.2 | `/health/ready` | без локального durable state; default + storage; signing/S3/broker files, DB env | +| PostgreSQL → postgres | 17.11, digest в Compose | pg_isready | postgres_data; default; локальный пароль env | +| Кэш → redis | 8.2.11, digest в Compose | authenticated PING | redis_data, AOF; default; локальный пароль env | +| LB → lb | HAProxy 3.2, digest в Compose | backend readiness | без volume данных; default; публичный localhost порт | +| S3 → s3 | SeaweedFS 4.52, digest в Compose | cluster status; bootstrap проверяет запись | s3_data; internal storage; credentials JSON file | +| Broker → broker | RabbitMQ 4.2.0, digest в Compose | rabbitmq-diagnostics check_running | broker_data; default; синтетические credentials | +| Worker → worker | образ приложения + ImageMagick | процесс/reconnect, состояние jobs | эфемерные ограниченные temp files; default + storage; secret files | +| Dispatcher → dispatcher | образ приложения | процесс/reconnect, состояние outbox | без локального durable state; default; broker secret file | + +У worker/dispatcher отключён унаследованный HTTP healthcheck: HTTP-сервер они не запускают. Состояние `running` само по себе не подтверждает прогресс; проверять нужно jobs/outbox и end-to-end сценарий. + +### Доставка и повторы + +- Durable RabbitMQ queue, persistent messages, publisher confirms и mandatory routing. Имя broker фиксировано, чтобы новый контейнер открывал ту же persistent node database; readiness ждёт запуска RabbitMQ application, а не только Erlang node. Consumer ack выполняется после устойчивого результата, постановки следующей попытки либо записи terminal failed. +- Сообщения соответствуют `jobs.schema.json`: UUID, версия, attempt, timestamp, опциональный trace carrier. Байтов файла, URL и credentials в них нет. Worker читает доверенные storage metadata из PostgreSQL. +- `job_id` стабилен для вложения и версии. Redelivery сама не увеличивает attempt; управляемый retry/recovery увеличивает его. По умолчанию пять попыток, exponential backoff с jitter 0.5–30 секунд в зависимости от номера попытки. +- Publisher имеет 20-секундный deadline на самом socket (одного контекста PublishWithContext недостаточно для ограничения сетевой записи). Dispatcher захватывает одну запись с lease 30 секунд, публикует её и фиксирует результат с проверкой token. Просроченный dispatcher не может изменить запись нового владельца. После пяти неудачных публикаций остаётся dead_letter-запись и наблюдаемый failed job. +- Worker: concurrency=2, prefetch=2, timeout=60s, lease=90s. Lease recovery работает независимо от consumer. Неактивный lease не продлевается: работа дольше лимита повторяется ограниченно. +- Постоянная ошибка заканчивается немедленно (`invalid_image`, `source_missing`, `source_corrupt`); временная повторяется и после исчерпания бюджета даёт `retry_exhausted`. API получает только безопасный error_code. +- Реестр failed — PostgreSQL. Replay явный, только для текущего failed job с доступным source; записывается аудит, номер attempt продолжает расти, выдаётся новый бюджет из пяти попыток. Уже успешную job replay не запускает. + +```sh +./scripts/compose.sh run --rm -T operator jobs list +./scripts/compose.sh run --rm -T operator jobs replay --id JOB_UUID +./scripts/compose.sh run --rm -T operator outbox replay --id JOB_UUID +./scripts/compose.sh run --rm -T operator jobs reprocess --id ATTACHMENT_UUID --pipeline-version 2 +``` + +`outbox replay` восстанавливает логическую failed job вместе с новой outbox-записью; ID аргумента — job, не отдельная доставка. Старые dead_letter-записи сохраняются для диагностики и не переводят новую попытку обратно в failed. + +### Версии и инварианты + +Manifest имеет PK `(attachment_id, pipeline_version, variant)`. Варианты публикуются одной DB-транзакцией только после записи всех объектов. Завершение проверяет текущую версию вложения, состояние job, token и действующий lease. Ready не откатывается из-за старой доставки. + +Объектные ключи включают вложение, версию, variant и SHA-256. Повтор одинакового эффекта использует тот же ключ; другая версия пишет в свой namespace. Поэтому устаревшая попытка не перезапишет новые байты S3, даже когда БД отклоняет её завершение. Незавершённые объекты возможны и удаляются как сироты после grace; они не становятся опубликованным вторым набором вариантов. + +Версия 1 использует WebP quality 80, версия 2 — 85. Остальные параметры одинаковы. Reprocess доступен для ready-вложения и строго большей поддерживаемой версии; на время новой работы статус queued и content=409. Сохраняются прежние manifest, новый оригинал побайтно совпадает с исходником. + +### Изображения и cleanup + +Thumbnail: JPEG, вписан в 256×256 без увеличения, размеры округляются вниз с минимумом 1 px. Crop: центральный квадрат JPEG 128×128 с разрешённым увеличением. Optimized: WebP внутри 1280×1280 без увеличения. Применяется EXIF orientation, alpha компонуется на белый, метаданные производных удаляются. JPEG quality=85. Не обещается уменьшение каждого файла. + +ImageMagick запускается без shell, отдельным дочерним процессом с отменой по timeout. Политика запрещает delegates/filters и разрешает только нужные coders; memory=256MiB, map=0, disk=256MiB, thread=1, time=60s, list-length=4. Worker ограничен 1 CPU/1GiB; его локальные файлы — только восстанавливаемые временные данные. API принимает максимум четыре одновременных upload и выполняет одну проверку декодирования на реплику. Пиксели проверяются до полной распаковки. Превышение ресурсов заканчивается ошибкой, а не бесконечным retry. + +Staging удаляется после устойчивого ready; failed source хранится 24h. Источники, которыми являются исторические опубликованные results, физически сохраняются. После удаления failed source replay отклоняется; оператор может восстановить оригинал с проверкой размера/SHA и затем явно повторить job. Reconcile защищает ссылки metadata/manifest и активные jobs, удаляет сироты только после grace=1h (>2×upload timeout). Старые originals lab4 переводятся миграцией на первую версию pipeline без изменения ID и ссылок сообщений. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Фактический вывод команд сохраняется отдельно от описания реализации. `evidence/pipeline.json` содержит повторяемые сценарии, UUID, длительности и результат каждого запуска; `evidence/images.json` — проверки изображений. Проверки защиты staging, удаления после ready, retention failed и восстановления исходника — `evidence/storage.json`. Остальные результаты — `evidence/checks.txt`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда / наблюдение | Доказательство | +| --- | --- | --- | +| Структура, схемы, сборка и анализ | make check; make validate; make build; make vet | checks.txt | +| Публичный контракт и предыдущие лабораторные | make test; make test-go; make test-storage | checks.txt | +| Сессия и данные после перезапуска; отказ API | test-persistence / test-failover с действительными hooks | checks.txt | +| 202 при остановленном worker, сообщение с queued-вложением, рестарт stores/broker и ready | worker_down_and_store_restart | pipeline.json | +| Опубликованное сообщение переживает пересоздание broker с тем же volume при остановленном dispatcher | make test-broker-persistence; hostname=broker сохраняет имя узла | broker-persistence.json | +| Thumbnail/crop/WebP, orientation, alpha, реальные размеры и hashes | student-tests/images.py | images.json | +| Сбой до/после S3 side effect и до/после COMMIT результата, до ack | crash_before_objects / after_original_put / after_objects / after_result_commit / before_ack | pipeline.json | +| Confirm получен, outbox ещё processing; повтор публикации безопасен | crash_confirm_before_outbox_mark | pipeline.json | +| Ready redelivery — no-op | ready_redelivery, 3 копии | pipeline.json | +| Реальный lease expiry, более новая попытка/версия, возобновление старого worker и поздняя старая доставка | lease_reorder_and_stale_version | pipeline.json | +| Staging защищён при queued; после ready удалён; failed retention и restore проверены | make test-staging | storage.json | +| Retry ограничен, failed наблюдаем, replay явный | bounded_retry_and_explicit_replay / permanent_error | pipeline.json | +| Сбой после staging и разрыв COMMIT/publish | api_crash_after_staging / api_crash_after_upload_commit | pipeline.json | -## Эксперименты и отказы +## Эксперимент трека 2 -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** атомарное сохранение намерения, durable доставка, fencing в БД и разделение версий/контента в S3 сохраняют принятые задания и предотвращают публикацию устаревшего эффекта при проверяемых crash/reorder/duplicate сценариях. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Методика: `make test-pipeline` включает исключительно для эксперимента `compose.faults.yaml`. Директивы в игнорируемом `.lab5-faults/` адресуются по job/attachment ID и точке сбоя. Барьер отмечает достижение точки; crash завершает процесс с кодом 86, restart policy поднимает его снова. Pause применяется один раз: другая попытка может завершиться после реального истечения lease. Общая директория не содержит токенов и очищается при завершении. -## Ограничения и следующие шаги +В эксперименте timeout=5s, lease=8s, concurrency/prefetch=2; остальные параметры не ускоряются искусственным изменением lease в БД. Окно после publisher confirm использует обычный outbox lease=30s. Для гарантированного повторного publish тест повторно делает свою outbox-запись pending; это внесение дубля, а не штатная операция пользователя. Для COMMIT/publish dispatcher заранее остановлен, проверяется pending-запись. Для crash до COMMIT отсутствие `202` не трактуется как потеря принятой работы. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Каждый сценарий выполняется три раза с новыми синтетическими пользователями/вложениями. Основной файл — публичный PNG fixture 320×200, 111817 bytes; дополнительных параллельных upload в fault-эксперименте нет. Измеряется длительность сценария (включает Docker stop/start и проверки), а не чистая latency обработки. Точность таймеров — monotonic clock клиента; нагрузочный максимум этим экспериментом не измеряется. + +Окружение выполнения: Apple M4 Pro, 14 CPU, host RAM 24GiB; Docker Desktop arm64, 14 CPU и около 7.75GiB RAM. CPU/RAM контейнеров ограничены Compose. Исследование проводится на одном Docker-host с сохранением volumes, без симуляции потери диска. + + +### Наблюдавшиеся результаты + +Все **39/39** сценариев трека завершились PASS (13 сценариев × 3 повтора). Интервалы ниже включают действия Docker и проверки; это не измерение чистого processing latency. + +| Сценарий | Минимум–максимум, с | +| --- | --- | +| worker_down_and_store_restart | 63.864–66.563 | +| crash_before_objects | 10.888–12.269 | +| crash_after_original_put | 11.222–12.171 | +| crash_after_objects | 11.174–11.340 | +| crash_after_result_commit | 1.973–2.088 | +| crash_before_ack | 2.017–2.098 | +| ready_redelivery | 2.435–3.002 | +| crash_confirm_before_outbox_mark | 33.988–36.021 | +| lease_reorder_and_stale_version | 14.652–15.225 | +| bounded_retry_and_explicit_replay | 13.737–15.238 | +| permanent_error | 2.552–2.699 | +| api_crash_after_staging | 0.144–0.817 | +| api_crash_after_upload_commit | 2.141–2.328 | + +После эксперимента отдельная проверка retention обнаружила и позволила исправить гонку восстановления исходника: Restore обновляет возраст source, а Cleanup берёт тот же advisory lock. Эта корректировка проверена `make test-staging`. После добавления socket deadline повторно выполнены confirm/crash, lease + stale version и COMMIT/publish на итоговом образе: **3/3 PASS**, см. `evidence/pipeline-final.json`. Image ID исходного полного fault-прогона сохранён в pipeline.json, итогового — в pipeline-final.json. + +## Ограничения + +At-least-once доставка сохраняется: publish/confirm/COMMIT/ack не образуют одну распределённую транзакцию. Доказывается идемпотентность опубликованного эффекта в перечисленных сценариях, а не exactly-once доставка и не отсутствие любых возможных отказов. + +Один PostgreSQL, S3, broker, LB и host — отдельные точки отказа. Durable очередь здесь не означает кластер RabbitMQ. Невосстановимая потеря volumes не покрывается; backup/restore инфраструктуры не реализован. После окончательной ошибки нужен операторский replay; при длительном отказе брокера принятые задания могут стать наблюдаемыми failed. Накопление outbox/jobs не ограничено глобальной квотой: ограничены upload, worker concurrency/prefetch и ресурсы, но отдельный трек перегрузки не заявляется. + +Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acks](https://www.rabbitmq.com/docs/confirms), [Go client](https://github.com/rabbitmq/amqp091-go), [ImageMagick resource policy](https://imagemagick.org/resources/). diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 0000000..75f47d8 --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,22 @@ +# Explicit isolated experiment profile. Never enabled by make up. +services: + api: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab5-faults:/faults + worker: + environment: + LAB5_FAULTS: "1" + JOB_TIMEOUT: 5s + JOB_LEASE: 8s + volumes: + - ./.lab5-faults:/faults + dispatcher: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab5-faults:/faults + operator: + environment: + LAB5_FAULTS: "1" diff --git a/compose.yaml b/compose.yaml index 217cce6..71bc0ca 100644 --- a/compose.yaml +++ b/compose.yaml @@ -40,6 +40,13 @@ x-api: &api S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT: ${JOB_TIMEOUT:-60s} + JOB_LEASE: ${JOB_LEASE:-90s} + STAGING_RETENTION: ${STAGING_RETENTION:-24h} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s} @@ -47,6 +54,7 @@ x-api: &api GOMEMLIMIT: 384MiB networks: [default, storage] volumes: + - ./secrets/lab5/broker_url:/run/secrets/broker_url:ro - ./secrets/lab5/private_key.pem:/run/certs/private_key.pem:ro - ./secrets/lab5/public_key.pem:/run/certs/public_key.pem:ro - ./secrets/lab5/s3_api_access:/run/secrets/s3_access:ro @@ -78,6 +86,50 @@ services: depends_on: postgres: condition: service_healthy + broker: + hostname: broker + image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524 + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh} + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, check_running] + interval: 3s + timeout: 5s + retries: 30 + stop_grace_period: 30s + worker: + <<: *api + command: [worker] + cpus: 1 + mem_limit: 1g + stop_grace_period: 10s + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + dispatcher: + <<: *api + command: [dispatcher] + cpus: 0.25 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy lb: image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0} restart: unless-stopped @@ -168,6 +220,7 @@ services: - ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro - ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro - ./tests/fixtures:/fixtures:ro + - ./secrets/lab5/broker_url:/run/secrets/broker_url:ro healthcheck: disable: true depends_on: {} @@ -176,6 +229,7 @@ volumes: postgres_data: redis_data: s3_data: + broker_data: networks: storage: diff --git a/evidence/broker-persistence.json b/evidence/broker-persistence.json new file mode 100644 index 0000000..84aa504 --- /dev/null +++ b/evidence/broker-persistence.json @@ -0,0 +1,10 @@ +{ + "result": "PASS", + "attachment_id": "cb1e8982-980d-4286-8224-335b644a5fc5", + "broker_hostname": "broker", + "old_container_id": "16848707eae7bdfb195da43c912ad66399678cc0a48d100bea119619f7e62009", + "new_container_id": "0dfeb1f83d39adc4de9c7798d65b5f125927fe95ae050e4a17e8a5238713b81b", + "message_present_before_and_after": true, + "outbox_published_before_replacement": true, + "dispatcher_stopped_during_replacement": true +} diff --git a/evidence/build.json b/evidence/build.json new file mode 100644 index 0000000..bdc85a1 --- /dev/null +++ b/evidence/build.json @@ -0,0 +1,14 @@ +{ + "date_utc": "2026-11-13T09:29:53.527687+00:00", + "base_commit": "ce035c85282e75513ea36cb429f4b8d775398d62", + "working_tree": "uncommitted solution", + "runtime_source_sha256": "cadfe8fc0d834a598d26e03d1c0348c907ac6fe9a4574aa959f156dfd69488d6", + "hash_method": "Sorted relative path + NUL + file content + NUL, for service/**/*.go, migrations/*.sql, go.mod, go.sum, Dockerfile and ImageMagick policy", + "image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f", + "go_local": "go version go1.27.0 darwin/arm64", + "docker_server": "29.7.2", + "compose": "Docker Compose version v5.3.1", + "docker_resources": "cpus=14 memory_bytes=8320954368 arch=aarch64", + "hardware": "Apple M4 Pro, 14 CPUs, 24 GiB host RAM", + "source_reuse": "Adapted core outbox from supplied mamont-optics project; original project left unchanged" +} diff --git a/evidence/checks.txt b/evidence/checks.txt new file mode 100644 index 0000000..faa5a76 --- /dev/null +++ b/evidence/checks.txt @@ -0,0 +1,201 @@ +Lab5 verification, UTC 2026-11-13T09:29:53.527687+00:00 +All commands below completed with exit code 0. +Go output excerpts omit SQL, runtime logs and synthetic request bodies. + +$ make check build vet; make validate PYTHON=.venv/bin/python +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +cd service && go build ./... +cd service && go build -o ../bin/benchmark ./cmd/benchmark +cd service && go vet ./... +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +$ make test +python3 tests/smoke.py --base-url "http://localhost:18085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +$ make test-go (race enabled) +? gitea.privcloud.tech/mtusi2026/cmd/app [no test files] +=== RUN TestLostPostResponseIsNotRetried +--- PASS: TestLostPostResponseIsNotRetried (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.461s +? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files] +=== RUN TestIdempotencyTransactionAndExpiry +PASS +ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.653s +? gitea.privcloud.tech/mtusi2026/internal/config [no test files] +? gitea.privcloud.tech/mtusi2026/internal/dto [no test files] +? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files] +=== RUN TestSessionFingerprints +--- PASS: TestSessionFingerprints (0.02s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.908s +? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files] +=== RUN TestVerifiedSessionIsRequestScoped +--- PASS: TestVerifiedSessionIsRequestScoped (0.00s) +=== RUN TestMissingTokenDoesNotCallSessionStore +--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 2.467s +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files] +? gitea.privcloud.tech/mtusi2026/migrations [no test files] +=== RUN TestInvalidJSON +=== RUN TestInvalidJSON/malformed +=== RUN TestInvalidJSON/wrong_type +=== RUN TestInvalidJSON/unknown_field +--- PASS: TestInvalidJSON (0.01s) + --- PASS: TestInvalidJSON/malformed (0.01s) + --- PASS: TestInvalidJSON/wrong_type (0.00s) + --- PASS: TestInvalidJSON/unknown_field (0.00s) +=== RUN TestForeignChatIsHidden +--- PASS: TestForeignChatIsHidden (0.54s) +=== RUN TestEmptyMessage +--- PASS: TestEmptyMessage (0.21s) +=== RUN TestDuplicateUsername +--- PASS: TestDuplicateUsername (0.28s) +=== RUN TestMessagePagination +--- PASS: TestMessagePagination (0.22s) +=== RUN TestAttachmentContractAndErrors +=== RUN TestAttachmentContractAndErrors/fake +=== RUN TestAttachmentContractAndErrors/mime_mismatch +=== RUN TestAttachmentContractAndErrors/wrong_type +=== RUN TestAttachmentContractAndErrors/truncated +--- PASS: TestAttachmentContractAndErrors (1.05s) + --- PASS: TestAttachmentContractAndErrors/fake (0.00s) + --- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s) + --- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s) + --- PASS: TestAttachmentContractAndErrors/truncated (0.00s) +=== RUN TestConcurrentAttachmentMessage +--- PASS: TestConcurrentAttachmentMessage (0.25s) +=== RUN TestConcurrentMessages +--- PASS: TestConcurrentMessages (1.84s) +=== RUN TestIdempotencyContract +--- PASS: TestIdempotencyContract (0.45s) +=== RUN TestConcurrentIdempotencyAcrossReplicas +--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s) +PASS +ok gitea.privcloud.tech/mtusi2026/tests 7.261s + +$ make test-storage (isolated database) +=== RUN TestIdempotencyTransactionAndExpiry +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object +=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock +--- PASS: TestIdempotencyTransactionAndExpiry (0.12s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.02s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s) +PASS + +$ make test-images +PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs + +$ make test-staging +PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified + +$ make test-broker-persistence +PASS: confirmed message survived broker container replacement with the same volume and stable node identity + +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.273} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +Actual process state before persistence hook +/mtusi2026lab5-api-1 pid=63585 started=2026-11-13T09:13:37.287792805Z +/mtusi2026lab5-api-2 pid=63443 started=2026-11-13T09:13:37.185017014Z +/mtusi2026lab5-broker-1 pid=16156 started=2026-11-13T09:06:10.180568751Z +/mtusi2026lab5-dispatcher-1 pid=63444 started=2026-11-13T09:13:37.187137055Z +/mtusi2026lab5-lb-1 pid=6588 started=2026-11-13T08:49:29.974451052Z +/mtusi2026lab5-postgres-1 pid=15734 started=2026-11-13T09:06:09.241419334Z +/mtusi2026lab5-redis-1 pid=15771 started=2026-11-13T09:06:09.289178667Z +/mtusi2026lab5-s3-1 pid=17699 started=2026-11-13T09:06:34.353438762Z +/mtusi2026lab5-worker-1 pid=66553 started=2026-11-13T09:14:11.339788001Z + +Actual process state after persistence hook +/mtusi2026lab5-api-1 pid=73635 started=2026-11-13T09:15:52.601737132Z +/mtusi2026lab5-api-2 pid=73510 started=2026-11-13T09:15:52.490649465Z +/mtusi2026lab5-broker-1 pid=70234 started=2026-11-13T09:14:51.740755173Z +/mtusi2026lab5-dispatcher-1 pid=73512 started=2026-11-13T09:15:52.489731548Z +/mtusi2026lab5-lb-1 pid=73700 started=2026-11-13T09:15:52.696327173Z +/mtusi2026lab5-postgres-1 pid=70113 started=2026-11-13T09:14:50.775761506Z +/mtusi2026lab5-redis-1 pid=70044 started=2026-11-13T09:14:50.718397506Z +/mtusi2026lab5-s3-1 pid=71476 started=2026-11-13T09:15:16.065232545Z +/mtusi2026lab5-worker-1 pid=73511 started=2026-11-13T09:15:52.490188673Z + +Actual state after API failover, before restoring stopped replica +api mtusi2026lab5-api-1 running healthy +api mtusi2026lab5-api-2 exited +broker mtusi2026lab5-broker-1 running healthy +dispatcher mtusi2026lab5-dispatcher-1 running +lb mtusi2026lab5-lb-1 running healthy +migrate mtusi2026lab5-migrate-1 exited +postgres mtusi2026lab5-postgres-1 running healthy +redis mtusi2026lab5-redis-1 running healthy +s3 mtusi2026lab5-s3-1 running healthy +s3-init mtusi2026lab5-s3-init-1 exited +worker mtusi2026lab5-worker-1 running + +Effective container resource limits +/mtusi2026lab5-api-1 memory_bytes=536870912 cpu_nano=1000000000 +/mtusi2026lab5-api-2 memory_bytes=536870912 cpu_nano=1000000000 +/mtusi2026lab5-broker-1 memory_bytes=536870912 cpu_nano=500000000 +/mtusi2026lab5-dispatcher-1 memory_bytes=134217728 cpu_nano=250000000 +/mtusi2026lab5-lb-1 memory_bytes=134217728 cpu_nano=500000000 +/mtusi2026lab5-postgres-1 memory_bytes=1073741824 cpu_nano=2000000000 +/mtusi2026lab5-redis-1 memory_bytes=134217728 cpu_nano=250000000 +/mtusi2026lab5-s3-1 memory_bytes=536870912 cpu_nano=1000000000 +/mtusi2026lab5-worker-1 memory_bytes=1073741824 cpu_nano=1000000000 + +make test-pipeline: 39/39 PASS; details and repetitions in pipeline.json +Final image targeted fault checks: 3/3 PASS; details in pipeline-final.json diff --git a/evidence/images.json b/evidence/images.json new file mode 100644 index 0000000..717ac91 --- /dev/null +++ b/evidence/images.json @@ -0,0 +1,53 @@ +{ + "results": [ + { + "case": "small", + "result": "PASS", + "thumbnail": [ + 16, + 8 + ] + }, + { + "case": "floor", + "result": "PASS", + "thumbnail": [ + 256, + 76 + ] + }, + { + "case": "alpha", + "result": "PASS", + "thumbnail": [ + 40, + 20 + ] + }, + { + "case": "orientation", + "result": "PASS", + "thumbnail": [ + 128, + 256 + ] + }, + { + "case": "thin", + "result": "PASS", + "thumbnail": [ + 1, + 256 + ] + }, + { + "case": "center_crop", + "result": "PASS", + "thumbnail": [ + 256, + 85 + ] + } + ], + "negative_cases": 4 +} diff --git a/evidence/pipeline-final.json b/evidence/pipeline-final.json new file mode 100644 index 0000000..e4293fd --- /dev/null +++ b/evidence/pipeline-final.json @@ -0,0 +1,42 @@ +{ + "date_utc": "2026-11-13T09:24:08.989020+00:00", + "commit": "ce035c85282e75513ea36cb429f4b8d775398d62", + "working_tree": "implementation changes uncommitted", + "repeats": 1, + "worker": { + "concurrency": 2, + "prefetch": 2, + "timeout_seconds": 5, + "lease_seconds": 8, + "memory": "1GiB", + "cpus": 1 + }, + "results": [ + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 1, + "result": "PASS", + "seconds": 33.527, + "job_id": "ce8a7658-c069-429a-b655-cb051f922e14" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 1, + "result": "PASS", + "seconds": 14.623, + "attachment_id": "6e709809-9ca2-4fb8-b7f6-155a3aace89c", + "old_job_id": "421fa4dc-3704-4f87-a490-3110f062459a", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 1, + "result": "PASS", + "seconds": 2.122, + "attachment_id": "293e92d6-4253-4fd9-8bad-719a488f6bdf", + "accepted_commit": true + } + ], + "image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f" +} diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 0000000..946326f --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,337 @@ +{ + "date_utc": "2026-11-13T09:09:05.756165+00:00", + "commit": "ce035c85282e75513ea36cb429f4b8d775398d62", + "working_tree": "implementation changes uncommitted", + "repeats": 3, + "worker": { + "concurrency": 2, + "prefetch": 2, + "timeout_seconds": 5, + "lease_seconds": 8, + "memory": "1GiB", + "cpus": 1 + }, + "results": [ + { + "scenario": "worker_down_and_store_restart", + "run": 1, + "result": "PASS", + "seconds": 66.563, + "attachment_id": "912b15f8-2ddb-4cbc-8db7-526540256749", + "job_id": "4913bc34-1533-4140-a0dc-c53efa5be0dd", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 1, + "result": "PASS", + "seconds": 10.888, + "job_id": "bfdb496b-f2e0-4836-9559-1a9b07ed21c4", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 1, + "result": "PASS", + "seconds": 12.171, + "job_id": "8468e628-5eef-42c2-9a86-d050d74cb672", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 1, + "result": "PASS", + "seconds": 11.174, + "job_id": "4c587b0f-fa79-4d9e-a3c7-4904db1e80d3", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 1, + "result": "PASS", + "seconds": 1.973, + "job_id": "419c9eeb-6467-4138-b05e-92f1bc929df8", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 1, + "result": "PASS", + "seconds": 2.017, + "job_id": "face4c1b-5597-4b24-8f61-a0e756e601de", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 1, + "result": "PASS", + "seconds": 2.435, + "job_id": "514c616d-367c-4693-8d33-60079b2ced87", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 1, + "result": "PASS", + "seconds": 33.996, + "job_id": "71f7c962-009e-4982-ae7b-e7f24fdcdc7f" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 1, + "result": "PASS", + "seconds": 14.652, + "attachment_id": "52a2e241-60a1-4443-8ef7-2e8e3e860577", + "old_job_id": "1c5824ca-774e-4230-9764-3e0938f4efeb", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 1, + "result": "PASS", + "seconds": 14.47, + "job_id": "8b8f8975-f4c8-45cf-80e9-00626c4e5c36", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 1, + "result": "PASS", + "seconds": 2.699, + "job_id": "e1948ce8-b194-4f66-8008-6e733ad42959" + }, + { + "scenario": "api_crash_after_staging", + "run": 1, + "result": "PASS", + "seconds": 0.817, + "attachment_id": "2d831ec0-5918-4290-8e26-3c2bec3cc0fd", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 1, + "result": "PASS", + "seconds": 2.328, + "attachment_id": "f0da85c6-003c-4d79-ba01-f3f961064741", + "accepted_commit": true + }, + { + "scenario": "worker_down_and_store_restart", + "run": 2, + "result": "PASS", + "seconds": 66.013, + "attachment_id": "64517fe4-6419-4257-bbf5-a47a88b51068", + "job_id": "873b9510-7ada-4dc0-bdfb-3e204ae2bc3d", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 2, + "result": "PASS", + "seconds": 11.241, + "job_id": "1c0ea968-87d2-45df-bb9b-d8b45c68dc1d", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 2, + "result": "PASS", + "seconds": 11.222, + "job_id": "748017a5-8f09-4741-8e35-35e4dbcb3c55", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 2, + "result": "PASS", + "seconds": 11.34, + "job_id": "489d2edd-b008-48fc-ba35-c9f64ee05ba2", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 2, + "result": "PASS", + "seconds": 2.088, + "job_id": "0ed57d8a-3fcc-42a4-a31f-2f02ae4f5f5f", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 2, + "result": "PASS", + "seconds": 2.026, + "job_id": "e9a8270c-3132-460a-a888-ee8722798e35", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 2, + "result": "PASS", + "seconds": 2.704, + "job_id": "b4f7d69b-3de5-4350-97bf-0e880ed94b0e", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 2, + "result": "PASS", + "seconds": 33.988, + "job_id": "5f7a89af-a045-423c-8c70-47b19615b577" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 2, + "result": "PASS", + "seconds": 14.694, + "attachment_id": "a1230ca9-8001-4bb4-b648-51446f646cbf", + "old_job_id": "2f1e1fbf-856d-4605-873c-b23011072a4e", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 2, + "result": "PASS", + "seconds": 13.737, + "job_id": "737209e2-5d3c-4693-8394-fc8173b88ee2", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 2, + "result": "PASS", + "seconds": 2.552, + "job_id": "e5e6dc90-9ea5-4d7c-aa11-c6c275d69793" + }, + { + "scenario": "api_crash_after_staging", + "run": 2, + "result": "PASS", + "seconds": 0.153, + "attachment_id": "ed8e2936-47fa-4717-9eb3-e63292c8a0ad", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 2, + "result": "PASS", + "seconds": 2.159, + "attachment_id": "a739f237-2278-421c-aa55-6024e717ad89", + "accepted_commit": true + }, + { + "scenario": "worker_down_and_store_restart", + "run": 3, + "result": "PASS", + "seconds": 63.864, + "attachment_id": "95176e6d-301a-4725-9861-7133507184eb", + "job_id": "e5f8b9fa-e460-4441-92b0-2af2ac2540bf", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 3, + "result": "PASS", + "seconds": 12.269, + "job_id": "67c70b6e-ea42-4c1d-894c-d2234a70bf03", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 3, + "result": "PASS", + "seconds": 11.402, + "job_id": "386f9b0d-1b1f-42b6-95a0-a13d6b97f118", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 3, + "result": "PASS", + "seconds": 11.259, + "job_id": "9c639085-f040-4989-90ba-54376c64630d", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 3, + "result": "PASS", + "seconds": 2.068, + "job_id": "73fe51b1-32eb-4c93-b8b0-0ed920a6641d", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 3, + "result": "PASS", + "seconds": 2.098, + "job_id": "2947da70-b658-48c0-9aa2-905bc0927781", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 3, + "result": "PASS", + "seconds": 3.002, + "job_id": "4f67af5a-1014-4f91-9a68-54739f2edde7", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 3, + "result": "PASS", + "seconds": 36.021, + "job_id": "c4135c71-64e7-44ba-9e4f-bf8f7489a459" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 3, + "result": "PASS", + "seconds": 15.225, + "attachment_id": "3bdce3c7-d7d5-49fb-9691-96e626317ae1", + "old_job_id": "dd3d35b5-14fe-420b-a650-e740bc43489c", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 3, + "result": "PASS", + "seconds": 15.238, + "job_id": "444a1bf1-42ed-4123-9ec7-7f614cd6d90f", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 3, + "result": "PASS", + "seconds": 2.604, + "job_id": "d21ee0e0-702d-4459-8fd6-5e5f62c6c11d" + }, + { + "scenario": "api_crash_after_staging", + "run": 3, + "result": "PASS", + "seconds": 0.144, + "attachment_id": "963d2034-021f-492f-81f3-d31431a2bc0f", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 3, + "result": "PASS", + "seconds": 2.141, + "attachment_id": "ea757ed8-98a8-4f3b-993e-556347380e26", + "accepted_commit": true + } + ], + "tested_image_id": "sha256:0b720b9bab11e4a762e8b180b26c290a43503c4db2db3436c201cb08725f24c0", + "runtime_source_sha256": "e6460140119a22471f992cd02c721b55463d0810a6a8e50333e193d674c7c79a", + "subsequent_fix": "Restore refreshes retention timestamp; Cleanup shares its source lock. Publisher has a physical socket deadline. Verified by storage.py and the additional final-image fault scenarios in pipeline-final.json; job processing, lease and manifest logic unchanged." +} diff --git a/evidence/storage.json b/evidence/storage.json new file mode 100644 index 0000000..8a953a0 --- /dev/null +++ b/evidence/storage.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "scenario": "queued_staging_protected", + "result": "PASS", + "attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b" + }, + { + "scenario": "staging_deleted_after_ready", + "result": "PASS", + "attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b" + }, + { + "scenario": "failed_retention_and_missing_source_replay", + "result": "PASS", + "age_simulated_hours": 25 + }, + { + "scenario": "verified_restore_then_replay", + "result": "PASS", + "job_id": "8d6f4591-e10f-4616-b387-fe7ec5020a84" + } + ] +} diff --git a/infra/imagemagick-policy.xml b/infra/imagemagick-policy.xml new file mode 100644 index 0000000..decc717 --- /dev/null +++ b/infra/imagemagick-policy.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/scripts/bootstrap-broker.py b/scripts/bootstrap-broker.py new file mode 100644 index 0000000..a1d7e47 --- /dev/null +++ b/scripts/bootstrap-broker.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Create synthetic local broker credentials, preserving existing settings.""" +from pathlib import Path +import secrets +from urllib.parse import quote +root=Path(__file__).resolve().parents[1] +env=root/'.env.lab5' +values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#')) +password=values.get('BROKER_PASSWORD') +if not password: + password=secrets.token_hex(24) + with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n') +p=root/'secrets/lab5/broker_url' +if p.exists(): p.chmod(0o600) +p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n') +p.chmod(0o444) +print('Local broker credentials ready.') diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 4c09f78..f67bf84 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -21,3 +21,5 @@ chmod 444 secrets/lab5/*.pem printf '%s\n' 'Local lab5 settings and synthetic keys ready; existing files preserved.' python3 scripts/bootstrap-storage.py + +python3 scripts/bootstrap-broker.py diff --git a/scripts/restart.sh b/scripts/restart.sh index 2469918..324c1ef 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -./scripts/compose.sh stop api lb -./scripts/compose.sh restart postgres redis s3 +./scripts/compose.sh stop api lb worker dispatcher +./scripts/compose.sh restart postgres redis s3 broker ./scripts/compose.sh run --rm -T s3-init -./scripts/compose.sh up -d --wait --wait-timeout 90 api lb +./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher diff --git a/service/cmd/app/main.go b/service/cmd/app/main.go index 804126d..353039d 100644 --- a/service/cmd/app/main.go +++ b/service/cmd/app/main.go @@ -27,6 +27,7 @@ import ( "gitea.privcloud.tech/mtusi2026/internal/service/attachment" "gitea.privcloud.tech/mtusi2026/internal/service/auth" "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" "gitea.privcloud.tech/mtusi2026/internal/service/user" "github.com/google/uuid" stdhttp "net/http" @@ -56,6 +57,9 @@ func run() error { if err != nil { return err } + if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") { + return runPipeline(cfg, attachmentConfig, os.Args[1:]) + } if len(os.Args) > 1 && os.Args[1] == "storage" { return runStorage(cfg, attachmentConfig, os.Args[2:]) } @@ -91,6 +95,12 @@ func run() error { chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache) objectStore := objectstore.New(attachmentConfig) attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc) + pipelineConfig, err := config.LoadPipeline() + if err != nil { + return err + } + outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil) + attachmentSvc.Configure(tm, outboxSvc, pipelineConfig) httpRouter := router.New( middleware.NewAuthMiddleware(authSvc), diff --git a/service/cmd/app/pipeline.go b/service/cmd/app/pipeline.go new file mode 100644 index 0000000..1cbabe4 --- /dev/null +++ b/service/cmd/app/pipeline.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitea.privcloud.tech/mtusi2026/internal/adapter/broker" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "log/slog" + "os" + "os/signal" + "syscall" + "time" +) + +func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error { + c, e := config.LoadPipeline() + if e != nil { + return e + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + pool := postgres.New(cfg.Postgres) + defer pool.Close() + tm := txmanager.New(pool) + repo := &storage.AttachmentsRepo{Pool: pool} + rabbit := &broker.Rabbit{URL: c.BrokerURL} + ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit) + svc := attachment.New(repo, objectstore.New(a), a, nil) + svc.Configure(tm, ob, c) + ob.OnDead = svc.PublishFailed + if args[0] == "jobs" || args[0] == "outbox" { + if len(args) < 2 { + return errors.New("operator action required") + } + fs := flag.NewFlagSet(args[1], flag.ContinueOnError) + idText := fs.String("id", "", "job ID (attachment ID for reprocess)") + version := fs.Int("pipeline-version", 2, "new pipeline version") + if e = fs.Parse(args[2:]); e != nil { + return e + } + if args[1] == "list" { + jobs, e := repo.ListJobs(ctx) + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(jobs) + } + id, e := uuid.Parse(*idText) + if e != nil { + return e + } + switch args[1] { + case "redeliver": + if os.Getenv("LAB5_FAULTS") != "1" { + return errors.New("redeliver is restricted to the explicit fault profile") + } + var payload []byte + e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + payload, e = json.Marshal(j.Message) + return e + }) + if e == nil { + e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload}) + } + case "replay": + e = svc.Replay(ctx, id) + case "reprocess": + e = svc.Reprocess(ctx, id, *version) + default: + return errors.New("unknown operator action") + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id}) + } + if c.BrokerURL == "" { + return errors.New("BROKER_URL_FILE is required") + } + if args[0] == "worker" { + // Recovery continues while the consumer is disconnected or a delivery is paused. + recoveryDone := make(chan struct{}) + defer func() { stop(); <-recoveryDone }() + go func() { + defer close(recoveryDone) + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if e := svc.Recover(ctx); e != nil { + slog.Warn("job_recovery_failed") + } + if e := svc.Cleanup(ctx); e != nil { + slog.Warn("source_cleanup_failed") + } + } + } + }() + for ctx.Err() == nil { + if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil { + slog.Warn("consumer_reconnecting") + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + return nil + } + for ctx.Err() == nil { + _, e = ob.ProcessOnce(ctx) + if e != nil { + slog.Warn("dispatcher_retry") + } + select { + case <-ctx.Done(): + case <-time.After(100 * time.Millisecond): + } + } + return nil +} diff --git a/service/cmd/app/storage.go b/service/cmd/app/storage.go index daeb6a1..09d459c 100644 --- a/service/cmd/app/storage.go +++ b/service/cmd/app/storage.go @@ -20,7 +20,9 @@ import ( "gitea.privcloud.tech/mtusi2026/internal/config" "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" ) func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { @@ -93,6 +95,12 @@ func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { pool := postgres.New(cfg.Postgres) defer pool.Close() svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil) + jobs, e := config.LoadPipeline() + if e != nil { + return e + } + tm := txmanager.New(pool) + svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs) switch args[0] { case "reconcile": fs := flag.NewFlagSet("reconcile", flag.ContinueOnError) diff --git a/service/go.mod b/service/go.mod index b816a85..ce30f28 100644 --- a/service/go.mod +++ b/service/go.mod @@ -52,6 +52,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect diff --git a/service/go.sum b/service/go.sum index 9c564cf..d7b4c21 100644 --- a/service/go.sum +++ b/service/go.sum @@ -94,6 +94,8 @@ github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/Ge github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE= github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= diff --git a/service/internal/adapter/broker/rabbitmq.go b/service/internal/adapter/broker/rabbitmq.go new file mode 100644 index 0000000..508a251 --- /dev/null +++ b/service/internal/adapter/broker/rabbitmq.go @@ -0,0 +1,170 @@ +package broker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + amqp "github.com/rabbitmq/amqp091-go" + "io" + "log/slog" + "net" + "regexp" + "time" +) + +const Queue = "attachments.v1" + +var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +type Rabbit struct{ URL string } + +func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) { + dial := amqp.DefaultDial(5 * time.Second) + if deadline > 0 { + dial = func(network, address string) (net.Conn, error) { + c, e := net.DialTimeout(network, address, 5*time.Second) + if e != nil { + return nil, e + } + if e = c.SetDeadline(time.Now().Add(deadline)); e != nil { + c.Close() + return nil, e + } + return c, nil + } + } + conn, e := amqp.DialConfig(r.URL, amqp.Config{Heartbeat: 10 * time.Second, Dial: dial}) + if e != nil { + return nil, nil, errors.New("broker connection unavailable") + } + ch, e := conn.Channel() + if e != nil { + conn.Close() + return nil, nil, errors.New("broker channel unavailable") + } + _, e = ch.QueueDeclare(Queue+".invalid", true, false, false, false, nil) + if e == nil { + e = ch.ExchangeDeclare("attachments.invalid", "direct", true, false, false, false, nil) + } + if e == nil { + e = ch.QueueBind(Queue+".invalid", "invalid", "attachments.invalid", false, nil) + } + if e == nil { + _, e = ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{"x-dead-letter-exchange": "attachments.invalid", "x-dead-letter-routing-key": "invalid"}) + } + if e != nil { + ch.Close() + conn.Close() + return nil, nil, errors.New("broker topology unavailable") + } + return conn, ch, nil +} +func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) error { + conn, ch, e := r.connect(20 * time.Second) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Confirm(false); e != nil { + return e + } + confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) + returns := ch.NotifyReturn(make(chan amqp.Return, 1)) + var job dto.JobMessage + if e = json.Unmarshal(m.Payload, &job); e != nil { + return e + } + fault.Hit(ctx, "before_publish", job.JobID) + if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: m.Payload}); e != nil { + return errors.New("broker publish failed") + } + select { + case c, ok := <-confirms: + if !ok || !c.Ack { + return errors.New("broker did not confirm") + } + select { + case <-returns: + return errors.New("unroutable job") + default: + } + case <-time.After(15 * time.Second): + return errors.New("confirm timeout") + case <-ctx.Done(): + return ctx.Err() + } + fault.Hit(ctx, "after_publish", job.JobID) + return nil +} +func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error { + conn, ch, e := r.connect(0) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Qos(prefetch, 0, false); e != nil { + return e + } + deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil) + if e != nil { + return e + } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + conn.Close() + case <-done: + } + }() + defer close(done) + tasks := make(chan amqp.Delivery) + finished := make(chan struct{}, concurrency) + for range concurrency { + go func() { + defer func() { finished <- struct{}{} }() + for d := range tasks { + var m dto.JobMessage + dec := json.NewDecoder(bytes.NewReader(d.Body)) + dec.DisallowUnknownFields() + e := dec.Decode(&m) + if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) { + _ = d.Reject(false) + continue + } + if e = process(ctx, m); e != nil { + slog.Warn("job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable") + select { + case <-ctx.Done(): + case <-time.After(200 * time.Millisecond): + } + _ = d.Nack(false, true) + continue + } + fault.Hit(ctx, "before_ack", m.JobID) + _ = d.Ack(false) + } + }() + } + for d := range deliveries { + select { + case tasks <- d: + case <-ctx.Done(): + } + } + close(tasks) + for range concurrency { + <-finished + } + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("broker consumer disconnected") +} diff --git a/service/internal/adapter/controller/http/attachmentHandler.go b/service/internal/adapter/controller/http/attachmentHandler.go index 577723c..d286d53 100644 --- a/service/internal/adapter/controller/http/attachmentHandler.go +++ b/service/internal/adapter/controller/http/attachmentHandler.go @@ -1,12 +1,14 @@ package http import ( + "gitea.privcloud.tech/mtusi2026/internal/dto" "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" "gitea.privcloud.tech/mtusi2026/internal/service/attachment" "io" + "mime" "net/http" "strconv" "strings" @@ -41,7 +43,20 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename")) return } - a, e := h.svc.Upload(w, r, user.UserID, id, values[0]) + if !attachment.ValidFilename(values[0]) { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename")) + return + } + mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) + if e != nil || (mt != "image/png" && mt != "image/jpeg") { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")) + return + } + if r.ContentLength > h.svc.Config.MaxBytes { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large")) + return + } + a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)}) if e != nil { if apperr.CodeFromError(e) == apperr.CodeTooManyRequests { w.Header().Set("Retry-After", "1") @@ -49,7 +64,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { framework.JSONError(r.Context(), w, e) return } - framework.JSONResponse(w, http.StatusCreated, a) + framework.JSONResponse(w, http.StatusAccepted, a) } func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { user, e := framework.ParseToken(r, h.provider) @@ -61,27 +76,32 @@ func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { if e != nil { return } - a, e := h.svc.Get(r.Context(), user.UserID, id) + + if !strings.HasSuffix(r.URL.Path, "/content") { + a, e := h.svc.Get(r.Context(), user.UserID, id) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, 200, a.Public) + return + } + v := r.URL.Query().Get("variant") + if v == "" { + v = "original" + } + if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) + return + } + obj, e := h.svc.Content(r.Context(), user.UserID, id, v) if e != nil { framework.JSONError(r.Context(), w, e) return } - if !strings.HasSuffix(r.URL.Path, "/content") { - framework.JSONResponse(w, 200, a.Public) - return - } - if v := r.URL.Query().Get("variant"); v != "" && v != "original" { - framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) - return - } - obj, e := h.svc.Store.Get(r.Context(), a.Key) - if e != nil { - framework.JSONError(r.Context(), w, attachment.Unavailable()) - return - } defer obj.Body.Close() - w.Header().Set("Content-Type", string(a.Public.ContentType)) - w.Header().Set("Content-Length", strconv.Itoa(a.Public.SizeBytes)) + w.Header().Set("Content-Type", obj.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10)) w.Header().Set("X-Content-Type-Options", "nosniff") // Once streaming has started a transport failure terminates the response; no JSON // error can be appended to an image. Content-Length exposes incomplete downloads. diff --git a/service/internal/adapter/repository/storage/attachment_scan.go b/service/internal/adapter/repository/storage/attachment_scan.go index 15fa213..5a10537 100644 --- a/service/internal/adapter/repository/storage/attachment_scan.go +++ b/service/internal/adapter/repository/storage/attachment_scan.go @@ -25,10 +25,15 @@ func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*Attachm } func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) { var found bool - e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)", key).Scan(&found) + e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found) return found, e } func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) { tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing) return tag.RowsAffected() == 1, e } + +func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error { + _, e := c.Exec(ctx, "SELECT 1") + return e +} diff --git a/service/internal/adapter/repository/storage/attachments.go b/service/internal/adapter/repository/storage/attachments.go index 6d35c74..0a3b2bb 100644 --- a/service/internal/adapter/repository/storage/attachments.go +++ b/service/internal/adapter/repository/storage/attachments.go @@ -8,40 +8,34 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" ) -type AttachmentRecord struct { - Public contracts.Attachment - Key string - Missing bool -} +type AttachmentRecord = dto.AttachmentRecord type AttachmentsRepo struct{ Pool *pgxpool.Pool } -const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing" +const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys" func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) { a := new(AttachmentRecord) p := &a.Public - if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing); err != nil { + var variants, keys []byte + if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil { return nil, apperr.WrapPostgres(err, "attachment unavailable") } - p.Status = "ready" - p.Variants = []contracts.Variant{} + if err := json.Unmarshal(variants, &p.Variants); err != nil { + return nil, err + } + if err := json.Unmarshal(keys, &a.VariantKeys); err != nil { + return nil, err + } return a, nil } func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) { return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id)) } -func (r *AttachmentsRepo) Save(ctx context.Context, conn *pgxpool.Conn, a *AttachmentRecord) error { - p := a.Public - _, e := conn.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.Key) - if e != nil { - return apperr.WrapPostgres(e, "failed to save attachment") - } - return nil -} // Session locks span S3 work and the database commit. A lost connection releases // the lock; the mandatory grace period also exceeds the maximum upload lifetime. diff --git a/service/internal/adapter/repository/storage/idempotency_test.go b/service/internal/adapter/repository/storage/idempotency_test.go index 2d151e2..db5164c 100644 --- a/service/internal/adapter/repository/storage/idempotency_test.go +++ b/service/internal/adapter/repository/storage/idempotency_test.go @@ -58,7 +58,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil { + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { t.Fatal(err) } if err = tx.Rollback(ctx); err != nil { @@ -75,7 +75,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - first, err := repo.CreateIdempotentTx(ctx, tx, req) + first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - second, err := repo.CreateIdempotentTx(ctx, tx, req) + second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = repo.CreateIdempotentTx(ctx, tx, req); err == nil { + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil { t.Fatal("expected injected failure") } if err = tx.Rollback(ctx); err != nil { @@ -134,7 +134,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil { + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { t.Fatal("transient failure was cached", err) } if err = tx.Commit(ctx); err != nil { @@ -257,7 +257,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = repo.CreateIdempotentTx(ctx, tx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) + _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) if err == nil { t.Fatal("expected missing attachment FK failure") } diff --git a/service/internal/adapter/repository/storage/jobs.go b/service/internal/adapter/repository/storage/jobs.go new file mode 100644 index 0000000..4bb9648 --- /dev/null +++ b/service/internal/adapter/repository/storage/jobs.go @@ -0,0 +1,134 @@ +package storage + +import ( + "context" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) } +func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) { + return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id)) +} +func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + p := a.Public + _, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status) + return e +} +func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + variants, e := json.Marshal(a.Public.Variants) + if e != nil { + return e + } + keys, e := json.Marshal(a.VariantKeys) + if e != nil { + return e + } + _, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing) + return e +} + +const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at` + +func scanJob(row pgx.Row) (*dto.AttachmentJob, error) { + j := new(dto.AttachmentJob) + m := &j.Message + err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt) + m.SchemaVersion = 1 + m.Type = "attachment.process.v1" + return j, err +} +func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) { + return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id)) +} +func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + m := j.Message + _, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt) + return e +} +func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + _, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt) + return e +} +func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) { + rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until>'attempt')::integer=j.attempt LIMIT 100`) + if e != nil { + return nil, e + } + defer rows.Close() + var ids []uuid.UUID + for rows.Next() { + var id uuid.UUID + if e = rows.Scan(&id); e != nil { + return nil, e + } + ids = append(ids, id) + } + return ids, rows.Err() +} +func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before) + if e != nil { + return nil, e + } + defer rows.Close() + var out []*dto.AttachmentRecord + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} + +func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error { + _, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size) + return e +} + +func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) { + var dead bool + e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead) + return dead, e +} diff --git a/service/internal/adapter/repository/storage/messages.go b/service/internal/adapter/repository/storage/messages.go index 61c29ba..0c1b6f9 100644 --- a/service/internal/adapter/repository/storage/messages.go +++ b/service/internal/adapter/repository/storage/messages.go @@ -1,10 +1,8 @@ package storage import ( - "bytes" "context" - "crypto/sha256" - "encoding/json" + "errors" "time" @@ -26,55 +24,49 @@ func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo { return &MessagesRepo{pool: pool} } -// CreateIdempotentTx serializes only requests sharing an idempotency scope. -// The transaction commits both the message and its replay record, or neither. -func (r *MessagesRepo) CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) { - scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey - if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil { - return nil, apperr.WrapPostgres(err, "failed to lock message key") +// LockKeyTx serializes persistence for one idempotency scope. +func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope) + if err != nil { + return apperr.WrapPostgres(err, "failed to lock message key") } - hash := messageHash(req) - var hashVersion int - var storedHash []byte - var messageID uuid.UUID - err := tx.QueryRow(ctx, `SELECT request_hash, message_id, hash_version FROM message_idempotency - WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, - req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&storedHash, &messageID, &hashVersion) - if err == nil { - if hashVersion == 1 && len(req.AttachmentIDs) == 0 { - hash = sha256.Sum256([]byte(req.Text)) - } else if hashVersion != 2 { - return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") - } - if !bytes.Equal(hash[:], storedHash) { - return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") - } - return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, messageID)) + return nil +} +func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) { + v := new(dto.MessageReplay) + err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil } - if !errors.Is(err, pgx.ErrNoRows) { + if err != nil { return nil, apperr.WrapPostgres(err, "failed to read message key") } - message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: time.Now().UTC().Truncate(time.Microsecond)} - if _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, - message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt); err != nil { + return v, nil +} +func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) { + return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id)) +} +func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error { + _, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt) + if err != nil { if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation { - return nil, apperr.Wrap(err, apperr.CodeNotFound, "chat not found") + return apperr.Wrap(err, apperr.CodeNotFound, "chat not found") } - return nil, apperr.WrapPostgres(err, "failed to create message") + return apperr.WrapPostgres(err, "failed to create message") } - for position, attachmentID := range req.AttachmentIDs { - if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, attachmentID, position); err != nil { - return nil, apperr.WrapPostgres(err, "failed to link attachment") + for position, id := range message.AttachmentIDs { + if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil { + return apperr.WrapPostgres(err, "failed to link attachment") } } - - if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) - VALUES($1,'POST',$2,$3,$4,$5,clock_timestamp()+interval '24 hours',2) - ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash, - message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash[:], message.ID); err != nil { - return nil, apperr.WrapPostgres(err, "failed to save message key") + return nil +} +func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error { + _, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires) + if err != nil { + return apperr.WrapPostgres(err, "failed to save message key") } - return &message, nil + return nil } func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) { @@ -120,20 +112,10 @@ func scanMessage(row pgx.Row) (*dto.Message, error) { const messageColumns = `id, chat_id, sender_id, text, created_at, ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)` -func messageHash(req dto.CreateMessage) [32]byte { - // Field order and a non-nil list define the canonical v2 representation. - canonical := struct { - Text string `json:"text"` - AttachmentIDs []uuid.UUID `json:"attachment_ids"` - }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} - encoded, _ := json.Marshal(canonical) // strings and UUIDs have no encoding failures. - return sha256.Sum256(encoded) -} - func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) { // Prevent a concurrent integrity update from invalidating the checked rows // until the message and its links have committed together. - rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) + rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) if err != nil { return nil, apperr.WrapPostgres(err, "failed to read attachments") } @@ -141,7 +123,7 @@ func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, i result := make([]dto.MessageAttachment, 0, len(ids)) for rows.Next() { var a dto.MessageAttachment - if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing); err != nil { + if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil { return nil, apperr.WrapPostgres(err, "failed to read attachment") } result = append(result, a) diff --git a/service/internal/adapter/repository/storage/outbox.go b/service/internal/adapter/repository/storage/outbox.go new file mode 100644 index 0000000..ffc3191 --- /dev/null +++ b/service/internal/adapter/repository/storage/outbox.go @@ -0,0 +1,52 @@ +package storage + +import ( + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +type OutboxRepo struct{ Pool *pgxpool.Pool } + +func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) { + _, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt) + return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e +} +func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) { + defer rows.Close() + for rows.Next() { + var m outbox.AcquiredMessage + if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil { + return + } + out = append(out, m) + } + err = rows.Err() + return +} +func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED) + UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String()) + if e != nil { + return nil, e + } + return scanOutbox(rows) +} +func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until 64 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = n + } + } + for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} { + if v := os.Getenv(k); v != "" { + d, e := time.ParseDuration(v) + if e != nil || d <= 0 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = d + } + } + if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency { + return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency") + } + if p := os.Getenv("BROKER_URL_FILE"); p != "" { + b, e := os.ReadFile(p) + if e != nil { + return nil, fmt.Errorf("read broker secret: %w", e) + } + c.BrokerURL = strings.TrimSpace(string(b)) + } + return c, nil +} diff --git a/service/internal/dto/attachment.go b/service/internal/dto/attachment.go index 76d3a17..1469e14 100644 --- a/service/internal/dto/attachment.go +++ b/service/internal/dto/attachment.go @@ -1 +1,48 @@ package dto + +import ( + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "github.com/google/uuid" + "io" + "time" +) + +type AttachmentRecord struct { + Public contracts.Attachment + Key string + StagingKey string + PipelineVersion int + VariantKeys map[string]string + Missing bool +} +type UploadAttachment struct { + UserID, ChatID uuid.UUID + Filename, ContentType string + Body io.Reader +} +type AttachmentContent struct { + Body io.ReadCloser + ContentType string + Size int64 +} +type JobMessage struct { + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` +} +type AttachmentJob struct { + Message JobMessage + State string + Token uuid.UUID + LeaseUntil *time.Time + NextAttemptAt time.Time + AttemptLimit int + ErrorCode *string + UpdatedAt time.Time +} diff --git a/service/internal/dto/message.go b/service/internal/dto/message.go index b85296d..48f33b6 100644 --- a/service/internal/dto/message.go +++ b/service/internal/dto/message.go @@ -35,4 +35,11 @@ type MessageAttachment struct { ID uuid.UUID ChatID uuid.UUID Missing bool + Status string +} + +type MessageReplay struct { + Hash []byte + HashVersion int + MessageID uuid.UUID } diff --git a/service/internal/gen/contracts/types.gen.go b/service/internal/gen/contracts/types.gen.go index 28d12db..977fa8c 100644 --- a/service/internal/gen/contracts/types.gen.go +++ b/service/internal/gen/contracts/types.gen.go @@ -22,7 +22,10 @@ const ( // Defines values for AttachmentStatus. const ( - Ready AttachmentStatus = "ready" + Failed AttachmentStatus = "failed" + Processing AttachmentStatus = "processing" + Queued AttachmentStatus = "queued" + Ready AttachmentStatus = "ready" ) // Defines values for ErrorErrorCode. @@ -52,14 +55,17 @@ const ( // Defines values for VariantName. const ( - Crop VariantName = "crop" - Optimized VariantName = "optimized" - Thumbnail VariantName = "thumbnail" + VariantNameCrop VariantName = "crop" + VariantNameOptimized VariantName = "optimized" + VariantNameThumbnail VariantName = "thumbnail" ) // Defines values for DownloadAttachmentParamsVariant. const ( - Original DownloadAttachmentParamsVariant = "original" + DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop" + DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized" + DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original" + DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail" ) // Attachment defines model for Attachment. diff --git a/service/internal/pkg/fault/fault.go b/service/internal/pkg/fault/fault.go new file mode 100644 index 0000000..0d5af15 --- /dev/null +++ b/service/internal/pkg/fault/fault.go @@ -0,0 +1,64 @@ +// Package fault implements explicit, file-controlled barriers for the isolated lab. +package fault + +import ( + "context" + "errors" + "github.com/google/uuid" + "os" + "path/filepath" + "strings" + "time" +) + +func Hit(ctx context.Context, point string, id uuid.UUID) { + if os.Getenv("LAB5_FAULTS") != "1" { + return + } + p := filepath.Join("/faults", point+"."+id.String()) + b, e := os.ReadFile(p) + if e != nil { + p = filepath.Join("/faults", point+".all") + b, e = os.ReadFile(p) + if e != nil { + return + } + } + hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if e != nil { + return + } + _, _ = hit.WriteString(id.String() + "\n") + _ = hit.Close() + switch strings.TrimSpace(string(b)) { + case "crash": + _ = os.Remove(p) + os.Exit(86) + case "pause": + for { + if _, e = os.Stat(p); os.IsNotExist(e) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + } +} + +func Error(point string, id uuid.UUID) error { + if os.Getenv("LAB5_FAULTS") != "1" { + return nil + } + b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String())) + if e != nil { + return nil + } + v := strings.TrimSpace(string(b)) + if v == "temporary" || v == "permanent" { + return errors.New(v) + } + return nil +} diff --git a/service/internal/pkg/imageprocessor/image.go b/service/internal/pkg/imageprocessor/image.go new file mode 100644 index 0000000..b87678c --- /dev/null +++ b/service/internal/pkg/imageprocessor/image.go @@ -0,0 +1,83 @@ +package imageprocessor + +import ( + "context" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "os" + "os/exec" + "path/filepath" + "time" +) + +func run(ctx context.Context, args ...string) error { + cmd := exec.CommandContext(ctx, "magick", args...) + cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60") + cmd.WaitDelay = time.Second + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("image conversion failed: %w", err) + } + return nil +} +func Validate(ctx context.Context, path string) error { + return run(ctx, "-regard-warnings", path, "null:") +} + +type Result struct { + Name, Path, MIME string + Width, Height int +} + +func fit(w, h, limit int) (int, int) { + if w <= limit && h <= limit { + return w, h + } + if w >= h { + return limit, max(1, h*limit/w) + } + return max(1, w*limit/h), limit +} +func Process(ctx context.Context, source, dir string, version int) ([]Result, error) { + normalized := filepath.Join(dir, "normalized.png") + if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil { + return nil, e + } + f, e := os.Open(normalized) + if e != nil { + return nil, e + } + cfg, _, e := image.DecodeConfig(f) + f.Close() + if e != nil { + return nil, e + } + tw, th := fit(cfg.Width, cfg.Height, 256) + ow, oh := fit(cfg.Width, cfg.Height, 1280) + out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}} + for _, v := range out { + args := []string{normalized} + if v.Name == "crop" { + side := min(cfg.Width, cfg.Height) + args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage") + } + args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality") + quality := "85" + if v.Name == "optimized" && version == 1 { + quality = "80" + } + coder := "jpeg:" + if v.Name == "optimized" { + coder = "webp:" + } + args = append(args, quality, coder+v.Path) + if e = run(ctx, args...); e != nil { + return nil, e + } + } + return out, nil +} diff --git a/service/internal/pkg/objectstore/s3.go b/service/internal/pkg/objectstore/s3.go index 5f90f28..78822da 100644 --- a/service/internal/pkg/objectstore/s3.go +++ b/service/internal/pkg/objectstore/s3.go @@ -67,3 +67,11 @@ func Missing(err error) bool { var api smithy.APIError return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404") } + +func (s *Store) List(ctx context.Context, token string) (*s3.ListObjectsV2Output, error) { + q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)} + if token != "" { + q.ContinuationToken = &token + } + return s.Client.ListObjectsV2(ctx, q) +} diff --git a/service/internal/service/attachment/interface.go b/service/internal/service/attachment/interface.go new file mode 100644 index 0000000..71599bf --- /dev/null +++ b/service/internal/service/attachment/interface.go @@ -0,0 +1,51 @@ +package attachment + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "io" + "time" +) + +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Outbox interface { + CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error) +} +type ObjectStore interface { + Put(context.Context, string, string, io.Reader, int64) error + Get(context.Context, string) (*s3.GetObjectOutput, error) + Head(context.Context, string) (*s3.HeadObjectOutput, error) + Delete(context.Context, string) error + Ping(context.Context) error + List(context.Context, string) (*s3.ListObjectsV2Output, error) +} +type Repository interface { + SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error + + Ping(context.Context) error + Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error) + GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error) + InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error) + InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + ExpiredJobs(context.Context) ([]uuid.UUID, error) + PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error) + FailedOutbox(context.Context) ([]uuid.UUID, error) + ListJobs(context.Context) ([]dto.AttachmentJob, error) + RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error) + AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error + Lock(context.Context, string) (*pgxpool.Conn, func(), error) + Exists(context.Context, *pgxpool.Conn, string) (bool, error) + Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error) + MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error) + CheckConn(context.Context, *pgxpool.Conn) error +} diff --git a/service/internal/service/attachment/pipeline.go b/service/internal/service/attachment/pipeline.go new file mode 100644 index 0000000..fcdb6cc --- /dev/null +++ b/service/internal/service/attachment/pipeline.go @@ -0,0 +1,486 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +var ErrBusy = errors.New("job lease held") + +func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) } +func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) } +func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + now := time.Now().UTC() + j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now} + if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil { + return e + } + return s.enqueue(ctx, tx, j) +} +func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + b, e := json.Marshal(j.Message) + if e != nil { + return e + } + _, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts}) + return e +} +func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) { + a, e := s.Get(ctx, user, id) + if e != nil { + return nil, e + } + if a.Public.Status != "ready" { + return nil, apperr.New(apperr.CodeConflict, "attachment not ready") + } + key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes) + if variant != "original" { + key = a.VariantKeys[variant] + if key == "" { + return nil, apperr.New(apperr.CodeNotFound, "variant not found") + } + for _, v := range a.Public.Variants { + if string(v.Name) == variant { + mime = string(v.ContentType) + size = int64(v.SizeBytes) + } + } + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + return nil, Unavailable() + } + return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil +} +func (s *Service) Process(ctx context.Context, m dto.JobMessage) error { + var j *dto.AttachmentJob + var a *dto.AttachmentRecord + claimed := false + err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + var e error + j, e = s.Repo.JobTx(ctx, tx, m.JobID) + if errors.Is(e, pgx.ErrNoRows) { + return nil + } + if e != nil { + return e + } + // Broker data is a notification. Trusted metadata and attempt live in PostgreSQL. + if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt { + return nil + } + a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" { + return nil + } + if j.State == "processing" { + return ErrBusy + } + if time.Now().Before(j.NextAttemptAt) { + return ErrBusy + } + j.State = "processing" + j.Token = uuid.New() + lease := time.Now().Add(s.Jobs.Lease) + j.LeaseUntil = &lease + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "processing" + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + claimed = true + return nil + }) + if err != nil || !claimed { + return err + } + s.Fault(ctx, "after_claim", m.JobID) + task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout) + defer cancel() + result, e := s.effects(task, j, a) + if e != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return s.finish(ctx, j, nil, e) + } + s.Fault(ctx, "after_objects", m.JobID) + if err = s.finish(ctx, j, result, nil); err != nil { + return err + } + s.Fault(ctx, "after_result_commit", m.JobID) + return nil +} + +type processed struct { + Key string + Variants []contracts.Variant + Keys map[string]string +} +type permanentError struct{ code string } + +func (e permanentError) Error() string { return e.code } +func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) { + dir, e := os.MkdirTemp("", "pipeline-*") + if e != nil { + return nil, e + } + defer os.RemoveAll(dir) + key := a.StagingKey + if key == "" { + key = a.Key + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + if objectstore.Missing(e) { + return nil, permanentError{"source_missing"} + } + return nil, e + } + defer obj.Body.Close() + path := filepath.Join(dir, "source") + f, e := os.Create(path) + if e != nil { + return nil, e + } + hash := sha256.New() + n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + f.Close() + if e != nil { + return nil, e + } + if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 { + return nil, permanentError{"source_corrupt"} + } + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Validate(ctx, f, string(a.Public.ContentType)) + f.Close() + if e != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, permanentError{"invalid_image"} + } + s.Fault(ctx, "before_objects", j.Message.JobID) + if e := fault.Error("before_objects", j.Message.JobID); e != nil { + if e.Error() == "permanent" { + return nil, permanentError{"invalid_image"} + } + return nil, e + } + converted, e := imageprocessor.Process(ctx, path, dir, j.Message.PipelineVersion) + if e != nil { + return nil, e + } + prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion) + result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}} + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n) + f.Close() + if e != nil { + return nil, e + } + s.Fault(ctx, "after_original_put", j.Message.JobID) + for _, v := range converted { + b, e := os.ReadFile(v.Path) + if e != nil { + return nil, e + } + sum := fmt.Sprintf("%x", sha256.Sum256(b)) + key := prefix + v.Name + "/" + sum + if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil { + return nil, e + } + result.Keys[v.Name] = key + result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum}) + } + return result, nil +} +func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + if cause != nil { + return s.retry(ctx, tx, j, a, cause) + } + + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil { + return e + } + for _, v := range result.Variants { + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil { + return e + } + } + j.State = "ready" + j.ErrorCode = nil + j.LeaseUntil = nil + j.UpdatedAt = time.Now().UTC() + a.Missing = false + a.Key = result.Key + a.VariantKeys = result.Keys + a.Public.Variants = result.Variants + a.Public.Status = "ready" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) + }) +} +func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error { + code := "processing_failed" + var permanent permanentError + terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit + if permanent.code != "" { + code = permanent.code + } + if terminal && permanent.code == "" { + code = "retry_exhausted" + } + j.LeaseUntil = nil + j.Token = uuid.New() + j.UpdatedAt = time.Now().UTC() + j.ErrorCode = &code + if terminal { + j.State = "failed" + a.Public.Status = "failed" + a.Public.ErrorCode = &code + } else { + j.State = "queued" + j.Message.Attempt++ + j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1)) + a.Public.Status = "queued" + a.Public.ErrorCode = nil + if e := s.enqueue(ctx, tx, j); e != nil { + return e + } + } + a.Public.UpdatedAt = j.UpdatedAt + if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) +} +func (s *Service) Recover(ctx context.Context) error { + ids, e := s.Repo.ExpiredJobs(ctx) + if e != nil { + return e + } + for _, id := range ids { + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + return s.retry(ctx, tx, j, a, errors.New("lease expired")) + }) + if e != nil { + return e + } + } + dead, e := s.Repo.FailedOutbox(ctx) + if e != nil { + return e + } + for _, id := range dead { + if e = s.PublishFailed(ctx, id); e != nil { + return e + } + } + return nil +} +func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "queued" { + return nil + } + dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt) + if e != nil { + return e + } + if !dead { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + code := "publish_failed" + j.State = "failed" + j.ErrorCode = &code + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "failed" + a.Public.ErrorCode = &code + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) + }) +} +func (s *Service) Replay(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion { + return apperr.New(apperr.CodeConflict, "only current failed job can be replayed") + } + source := a.StagingKey + if source == "" { + return errors.New("source unavailable; restore original first") + } + if _, e = s.Store.Head(ctx, source); e != nil { + return errors.New("source unavailable") + } + j.State = "queued" + j.Message.Attempt++ + j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1 + j.NextAttemptAt = time.Now().UTC() + j.UpdatedAt = j.NextAttemptAt + j.Token = uuid.New() + j.ErrorCode = nil + j.LeaseUntil = nil + a.Public.Status = "queued" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil { + return e + } + return s.enqueue(ctx, tx, j) + }) +} +func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error { + if version < 1 || version > 2 { + return errors.New("supported pipeline versions: 1, 2") + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if version <= a.PipelineVersion || a.Public.Status != "ready" { + return errors.New("requires ready attachment and newer version") + } + a.PipelineVersion = version + a.StagingKey = a.Key + a.Public.Status = "queued" + a.Public.Variants = []contracts.Variant{} + a.VariantKeys = map[string]string{} + a.Public.UpdatedAt = time.Now().UTC() + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }) +} +func (s *Service) Cleanup(ctx context.Context) error { + records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention)) + if e != nil { + return e + } + for _, record := range records { + // Serialize deletion with operator Restore and the initial upload handoff. + _, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey) + if lockErr != nil { + return lockErr + } + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, record.Public.Id) + if e != nil { + return e + } + if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" { + return nil + } + if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) { + return nil + } + if !strings.HasPrefix(a.StagingKey, "results/") { + if e = s.Store.Delete(ctx, a.StagingKey); e != nil { + return e + } + } + a.StagingKey = "" + return s.Repo.UpdateTx(ctx, tx, a) + }) + unlock() + if e != nil { + return e + } + } + return nil +} +func managedPipelineKey(key string) bool { + return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/") +} diff --git a/service/internal/service/attachment/reconcile.go b/service/internal/service/attachment/reconcile.go index 0a11258..0ad54ad 100644 --- a/service/internal/service/attachment/reconcile.go +++ b/service/internal/service/attachment/reconcile.go @@ -10,8 +10,9 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" ) @@ -38,17 +39,17 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, result.Apply = apply result.GraceSeconds = s.Config.Grace.Seconds() defer func() { result.DurationMS = time.Since(start).Milliseconds() }() - if err = s.Repo.Pool.Ping(ctx); err != nil { + if err = s.Repo.Ping(ctx); err != nil { return } result.S3Requests++ if err = s.Store.Ping(ctx); err != nil { return } - pager := s3.NewListObjectsV2Paginator(s.Store.Client, &s3.ListObjectsV2Input{Bucket: &s.Store.Bucket, Prefix: aws.String(Prefix), MaxKeys: aws.Int32(100)}) - for pager.HasMorePages() { + token := "" + for { result.S3Requests++ - page, e := pager.NextPage(ctx) + page, e := s.Store.List(ctx, token) if e != nil { return result, e } @@ -92,7 +93,7 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, return nil } result.DBChecks++ - if _, e = conn.Exec(ctx, "SELECT 1"); e != nil { + if e = s.Repo.CheckConn(ctx, conn); e != nil { return e } result.S3Requests++ @@ -106,7 +107,12 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, return result, e } } + if !aws.ToBool(page.IsTruncated) { + break + } + token = aws.ToString(page.NextContinuationToken) } + after := uuid.Nil for { result.DBChecks++ @@ -120,6 +126,9 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, for _, a := range page { result.Metadata++ after = a.Public.Id + if a.Public.Status != "ready" { + continue + } result.LockAttempts++ conn, unlock, e := s.Repo.Lock(ctx, a.Key) if e != nil { @@ -201,5 +210,19 @@ func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error return fmt.Errorf("stored original verification failed") } _, e = s.Repo.MarkMissing(ctx, conn, id, false) - return e + if e != nil { + return e + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + current, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if current.Public.Status == "failed" { + current.StagingKey = current.Key + current.Public.UpdatedAt = time.Now().UTC() + return s.Repo.UpdateTx(ctx, tx, current) + } + return nil + }) } diff --git a/service/internal/service/attachment/service.go b/service/internal/service/attachment/service.go index 36c5d71..686bc13 100644 --- a/service/internal/service/attachment/service.go +++ b/service/internal/service/attachment/service.go @@ -8,8 +8,7 @@ import ( _ "image/jpeg" _ "image/png" "io" - "mime" - "net/http" + "os" "strings" "time" @@ -18,28 +17,31 @@ import ( "github.com/google/uuid" - "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" "gitea.privcloud.tech/mtusi2026/internal/config" "gitea.privcloud.tech/mtusi2026/internal/dto" "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" - "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "github.com/jackc/pgx/v5" ) const Prefix = "originals/" +const StagingPrefix = "staging/" type Membership interface { GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error) } type Service struct { - Repo *storage.AttachmentsRepo - Store *objectstore.Store + Repo Repository + Store ObjectStore Config *config.Attachment chats Membership + tm TransactionManager + outbox Outbox + Jobs *config.Pipeline uploads, decoders chan struct{} } -func New(repo *storage.AttachmentsRepo, store *objectstore.Store, c *config.Attachment, chats Membership) *Service { +func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service { return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)} } func ValidFilename(s string) bool { @@ -53,21 +55,15 @@ func ValidFilename(s string) bool { } return true } -func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid.UUID, filename string) (*contracts.Attachment, error) { - ctx := r.Context() - if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: chat}); e != nil { +func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) { + s.tm = tm + s.outbox = outbox + s.Jobs = jobs +} +func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (*contracts.Attachment, error) { + if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil { return nil, e } - if !ValidFilename(filename) { - return nil, apperr.New(apperr.CodeBadRequest, "invalid filename") - } - mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) - if e != nil || (mt != "image/png" && mt != "image/jpeg") { - return nil, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG") - } - if r.ContentLength > s.Config.MaxBytes { - return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") - } select { case s.uploads <- struct{}{}: defer func() { <-s.uploads }() @@ -80,23 +76,22 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid } defer os.Remove(f.Name()) defer f.Close() - r.Body = http.MaxBytesReader(w, r.Body, s.Config.MaxBytes) hash := sha256.New() - size, e := io.Copy(io.MultiWriter(f, hash), r.Body) + size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1)) if e != nil { - if _, ok := e.(*http.MaxBytesError); ok { - return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") - } return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload") } - if e = s.Validate(ctx, f, mt); e != nil { + if size > s.Config.MaxBytes { + return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") + } + if e = s.Validate(ctx, f, req.ContentType); e != nil { return nil, e } now := time.Now().UTC().Truncate(time.Microsecond) id := uuid.New() - key := Prefix + id.String() - a := &storage.AttachmentRecord{Key: key, Public: contracts.Attachment{Id: id, ChatId: chat, OwnerId: user, Filename: filename, ContentType: contracts.AttachmentContentType(mt), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "ready", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} - conn, unlock, e := s.Repo.Lock(ctx, key) + key := StagingPrefix + id.String() + a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} + _, unlock, e := s.Repo.Lock(ctx, key) if e != nil { return nil, e } @@ -104,14 +99,19 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid if _, e = f.Seek(0, 0); e != nil { return nil, e } - if e = s.Store.Put(ctx, key, mt, f, size); e != nil { + if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil { return nil, Unavailable() } - // Never blindly compensate a failed/ambiguous COMMIT with DELETE: a committed - // row may already reference this key. Reconciliation decides after grace. - if e = s.Repo.Save(ctx, conn, a); e != nil { + s.Fault(ctx, "after_staging", id) + if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if e := s.Repo.InsertTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }); e != nil { return nil, e } + s.Fault(ctx, "after_upload_commit", id) return &a.Public, nil } func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { @@ -134,12 +134,12 @@ func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { if _, e = f.Seek(0, 0); e != nil { return e } - if _, _, e = image.Decode(f); e != nil { + if e = validateImage(ctx, f.Name()); e != nil { return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image") } return ctx.Err() } -func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.AttachmentRecord, error) { +func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) { a, e := s.Repo.Get(ctx, id) if e != nil { return nil, e @@ -154,6 +154,9 @@ func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.Attachm } func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } func ManagedKey(key string) bool { + if managedPipelineKey(key) { + return true + } if !strings.HasPrefix(key, Prefix) { return false } diff --git a/service/internal/service/chat/idempotency.go b/service/internal/service/chat/idempotency.go new file mode 100644 index 0000000..13fa962 --- /dev/null +++ b/service/internal/service/chat/idempotency.go @@ -0,0 +1,54 @@ +package chat + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +// CreateIdempotentTx performs the message use case inside an existing transaction. +func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) { + scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey + if err := repo.LockKeyTx(ctx, tx, scope); err != nil { + return nil, err + } + canonical := struct { + Text string `json:"text"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` + }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} + encoded, err := json.Marshal(canonical) + if err != nil { + return nil, err + } + hash := sha256.Sum256(encoded) + replay, err := repo.ReplayTx(ctx, tx, req) + if err != nil { + return nil, err + } + if replay != nil { + if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 { + hash = sha256.Sum256([]byte(req.Text)) + } else if replay.HashVersion != 2 { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + if !bytes.Equal(hash[:], replay.Hash) { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + return repo.MessageTx(ctx, tx, replay.MessageID) + } + now := time.Now().UTC().Truncate(time.Microsecond) + message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now} + if err = repo.InsertMessageTx(ctx, tx, message); err != nil { + return nil, err + } + if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil { + return nil, err + } + return &message, nil +} diff --git a/service/internal/service/chat/interface.go b/service/internal/service/chat/interface.go index dbe7157..17c7742 100644 --- a/service/internal/service/chat/interface.go +++ b/service/internal/service/chat/interface.go @@ -2,6 +2,7 @@ package chat import ( "context" + "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -17,7 +18,11 @@ type chatsRepo interface { type messagesRepo interface { AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) - CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) + LockKeyTx(context.Context, pgx.Tx, string) error + ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error) + MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error) + InsertMessageTx(context.Context, pgx.Tx, dto.Message) error + SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) } diff --git a/service/internal/service/chat/svc.go b/service/internal/service/chat/svc.go index 6dacf96..700f168 100644 --- a/service/internal/service/chat/svc.go +++ b/service/internal/service/chat/svc.go @@ -190,6 +190,9 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M if attachment.ChatID != req.ChatID { return apperr.New(apperr.CodeNotFound, "attachment not found") } + if attachment.Status == "failed" { + return apperr.New(apperr.CodeConflict, "failed attachment") + } if attachment.Missing { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } @@ -197,7 +200,7 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M } var err error - message, err = c.messagesRepo.CreateIdempotentTx(ctx, tx, req) + message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req) return err }) return message, err diff --git a/service/internal/service/outbox/api.go b/service/internal/service/outbox/api.go new file mode 100644 index 0000000..aaac1a7 --- /dev/null +++ b/service/internal/service/outbox/api.go @@ -0,0 +1,12 @@ +package outbox + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +type API interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + Create(context.Context, CreateMessageReq) (CreateMessageRes, error) +} diff --git a/service/internal/service/outbox/models.go b/service/internal/service/outbox/models.go new file mode 100644 index 0000000..fc90e0e --- /dev/null +++ b/service/internal/service/outbox/models.go @@ -0,0 +1,115 @@ +package outbox + +import ( + "time" + + "github.com/google/uuid" +) + +type CreateMessageReq struct { + ID uuid.UUID + CreatedAt time.Time + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + IdempotencyKey *string + SchemaVersion int + CausationID *uuid.UUID + Metadata map[string]string + Payload []byte + MaxAttempts int + Status Status +} + +type CreateMessageRes struct { + ID uuid.UUID + CreatedAt time.Time +} + +type MarkPublishedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + PublishedAt time.Time +} + +type MarkFailedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string +} + +type MarkDeadLetterMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string +} + +type AcquireMessagesForProcessingReq struct { + Limit uint + LockDuration time.Duration +} + +type AcquiredMessage struct { + Token uuid.UUID + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Status Status + Metadata map[string]string + Payload []byte + Attempts int + MaxAttempts int + LastError *string + LockedAt *time.Time + LockedUntil *time.Time + NextAttemptAt *time.Time + PublishedAt *time.Time + DeadLetterAt *time.Time + CreatedAt time.Time + UpdatedAt *time.Time +} + +type RecoverStuckMessagesAsFailedReq struct { + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string + Limit uint +} + +type RecoverStuckMessagesAsDeadLetterReq struct { + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string + Limit uint +} + +type DeliveryMessage struct { + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Payload []byte + Metadata map[string]string +} + +type ProcessOnceResult struct { + RecoveredAsFailed int64 + RecoveredAsDeadLetter int64 + + Acquired int + Published int + Failed int + DeadLettered int +} diff --git a/service/internal/service/outbox/status.go b/service/internal/service/outbox/status.go new file mode 100644 index 0000000..c7a1d3a --- /dev/null +++ b/service/internal/service/outbox/status.go @@ -0,0 +1,16 @@ +package outbox + +const ( + StatusCreated Status = "created" + StatusPending Status = "pending" + StatusProcessing Status = "processing" + StatusPublished Status = "published" + StatusFailed Status = "failed" + StatusDeadLetter Status = "dead_letter" +) + +type Status string + +func (s Status) String() string { + return string(s) +} diff --git a/service/internal/service/outbox/svc.go b/service/internal/service/outbox/svc.go new file mode 100644 index 0000000..7dd7faa --- /dev/null +++ b/service/internal/service/outbox/svc.go @@ -0,0 +1,114 @@ +// Adapted from mamont-optics/application/internal/service/outbox. +// Retains transactional creation, batch acquisition, bounded retries and lease recovery. +package outbox + +import ( + "context" + "encoding/json" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "math/rand/v2" + "time" +) + +type Repository interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error) + Expired(context.Context) ([]AcquiredMessage, error) + Update(context.Context, AcquiredMessage, Status, time.Time, string) error +} +type TransportPublisher interface { + Publish(context.Context, DeliveryMessage) error +} +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Svc struct { + repo Repository + tm TransactionManager + publisher TransportPublisher + OnDead func(context.Context, uuid.UUID) error +} + +func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc { + return &Svc{repo: repo, tm: tm, publisher: publisher} +} +func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) { + if r.ID == uuid.Nil { + r.ID = uuid.New() + } + if r.CreatedAt.IsZero() { + r.CreatedAt = time.Now().UTC() + } + if r.Status == "" { + r.Status = StatusPending + } + if r.MaxAttempts == 0 { + r.MaxAttempts = 5 + } + return s.repo.CreateTx(ctx, tx, r) +} +func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) { + err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e }) + return +} +func Backoff(attempt int) time.Duration { + n := min(max(attempt-1, 0), 5) + cap := min(time.Second*time.Duration(1<= m.MaxAttempts { + status = StatusDeadLetter + } + if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil { + return err + } + if status == StatusDeadLetter && s.OnDead != nil { + var j struct { + JobID uuid.UUID `json:"job_id"` + } + if err := json.Unmarshal(m.Payload, &j); err != nil { + return err + } + return s.OnDead(ctx, j.JobID) + } + return nil +} diff --git a/service/internal/service/outbox/transport.go b/service/internal/service/outbox/transport.go new file mode 100644 index 0000000..a92608c --- /dev/null +++ b/service/internal/service/outbox/transport.go @@ -0,0 +1,7 @@ +package outbox + +type Transport string + +const TransportAMQP Transport = "amqp" + +func (t Transport) String() string { return string(t) } diff --git a/service/migrations/008_async_pipeline.sql b/service/migrations/008_async_pipeline.sql new file mode 100644 index 0000000..4607e20 --- /dev/null +++ b/service/migrations/008_async_pipeline.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')), + ADD COLUMN error_code text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text NOT NULL DEFAULT '', + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}'; +UPDATE attachments SET staging_key=object_key; +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0), + attempt_limit integer NOT NULL, state text NOT NULL, + token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL, + error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until); +CREATE TABLE outbox ( + id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + payload jsonb NOT NULL, status text NOT NULL, + attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL, + token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL, + last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL +); +CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at); +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- Existing lab4 originals become durable sources for the first asynchronous run. +INSERT INTO attachment_jobs + SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments; +INSERT INTO outbox + SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at), + 'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs; + +---- create above / drop below ---- +DROP TABLE job_audit; +DROP TABLE outbox; +DROP TABLE attachment_jobs; +ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys; diff --git a/service/migrations/009_variant_manifest.sql b/service/migrations/009_variant_manifest.sql new file mode 100644 index 0000000..84413c8 --- /dev/null +++ b/service/migrations/009_variant_manifest.sql @@ -0,0 +1,8 @@ +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL, + variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')), + object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL, + PRIMARY KEY(attachment_id,pipeline_version,variant) +); +---- create above / drop below ---- +DROP TABLE attachment_variants; diff --git a/service/tests/attachments_test.go b/service/tests/attachments_test.go index e9a5860..1a3e6bf 100644 --- a/service/tests/attachments_test.go +++ b/service/tests/attachments_test.go @@ -69,11 +69,12 @@ func TestAttachmentContractAndErrors(t *testing.T) { Sha string `json:"sha256"` Size int `json:"size_bytes"` } - decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 201, &att) + decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att) hash := sha256.Sum256(data) if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) { t.Fatal("original metadata mismatch") } + waitAttachmentReady(t, api, alice, att.ID) content := "/api/v1/attachments/" + att.ID + "/content" got := api.request(t, "GET", content, bob, "") if got.status != 200 || !bytes.Equal(got.body, data) { @@ -115,7 +116,7 @@ func TestAttachmentContractAndErrors(t *testing.T) { boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk)) boundary = append(boundary, data[len(data)-12:]...) var boundaryAtt map[string]any - decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 201, &boundaryAtt) + decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt) oversized := make([]byte, 10485761) copy(oversized, data) @@ -127,7 +128,7 @@ func TestAttachmentContractAndErrors(t *testing.T) { t.Fatal(e) } var jpegAtt map[string]any - decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 201, &jpegAtt) + decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt) imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}` var message map[string]any key := uuid.NewString() @@ -145,7 +146,9 @@ func TestAttachmentContractAndErrors(t *testing.T) { for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} { assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request") } - assertError(t, api.request(t, "GET", content+"?variant=thumbnail", bob, ""), 400, "invalid_request") + if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 { + t.Fatal("thumbnail unavailable") + } } func TestConcurrentAttachmentMessage(t *testing.T) { api := newAPI(t) @@ -154,7 +157,7 @@ func TestConcurrentAttachmentMessage(t *testing.T) { var att struct { ID string `json:"id"` } - decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 201, &att) + decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att) key := uuid.NewString() body := `{"text":"","attachment_ids":["` + att.ID + `"]}` type result struct { @@ -202,3 +205,22 @@ func TestConcurrentAttachmentMessage(t *testing.T) { t.Fatal("duplicate history") } } + +func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + var att struct { + Status string `json:"status"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att) + if att.Status == "ready" { + return + } + if att.Status == "failed" { + t.Fatal("attachment failed") + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("attachment readiness deadline") +} diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..6e922fb --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,14 @@ +# Проверки решения lab5 + +Публичные `tests/` не изменены. Собственные проверки используют их HTTP-клиент и создают только синтетические данные в локальном стенде. + +- `make test-images`: реальные форматы и размеры вариантов, floor/no-upscale, EXIF orientation, белый фон alpha, центральный crop, hashes, ACL и отрицательные входы. +- `make test-staging`: защита queued-source, удаление staging после ready, terminal source_missing, 24h retention и восстановление с явным replay. Для retention меняется только возраст одной синтетической записи на 25h; реальный день не ожидается и конфигурация retention не сокращается. +- `make test-broker-persistence`: заменяет контейнер брокера при остановленных worker и dispatcher; проверяет, что уже опубликованное сообщение осталось в очереди с тем же volume. +- `make test-pipeline`: 13 сценариев × 3 повтора, включая crash, дубли, lease recovery, reorder, устаревший worker и версию pipeline. На время выполнения подключает `compose.faults.yaml`, перезапускает процессы и stores с сохранением volumes. В конце восстанавливает обычный Compose без fault injection. + +Перед запуском нужен `make up`. Проверки, которые останавливают/перезапускают процессы (`test-staging`, `test-pipeline`, persistence/failover), запускаются последовательно. Результаты — `evidence/*.json`; секреты и токены туда не записываются. Директивы fault injection лежат в игнорируемой `.lab5-faults/`. + +`jobs redeliver --id JOB_UUID` — только инструмент экспериментального профиля с `LAB5_FAULTS=1`, публикующий дополнительную копию сохранённой job. Обычный оператор использует `jobs replay` для failed или `jobs reprocess` для новой версии. + +Исследование и ограничения описаны в [REPORT.md](../REPORT.md). diff --git a/student-tests/broker_persistence.py b/student-tests/broker_persistence.py new file mode 100644 index 0000000..6d63bc9 --- /dev/null +++ b/student-tests/broker_persistence.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a confirmed queued message survives broker container replacement, not only restart.""" +from pathlib import Path +import json +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check + +def compose(*args): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150) + check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip() + +def queued(): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30) + if r.returncode:return 0 + out=r.stdout + for line in out.splitlines(): + fields=line.split() + if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1]) + return 0 + +def main(): + c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap() + try: + compose('stop','worker');att,data=c.attachment(alice,chat) + deadline=time.monotonic()+20 + while queued()<1 and time.monotonic()=1,'message not durably queued before replacement') + published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id']) + check(published=='t','outbox not marked published before replacement') + compose('stop','dispatcher') + old=compose('ps','-q','broker') + compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker') + new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced') + deadline=time.monotonic()+30 + while queued()<1 and time.monotonic()=1,'confirmed queue message lost after replacement') + compose('start','worker');c.wait_ready(bob,att) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement') + result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True} + (ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n') + print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity') + finally:compose('start','worker','dispatcher') +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..4b32463 --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Image contract checks using synthetic fixtures and real decoding in ImageMagick.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys +import urllib.parse +import zlib +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +from image_probe import jpeg_size + +def magick(args,data=None): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30) + check(r.returncode==0,'fixture decoder failed') + return r.stdout + +def png(w,h,rgba): + def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b)) + return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'') + +def main(): + c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[] + jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255))) + # EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise. + tiff=b'II'+struct.pack('H',len(exif)+2)+exif+jpeg[2:] + for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]: + att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att) + original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed') + for v in ready['variants']: + b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch') + actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split())) + check(actual==(v['width'],v['height']),'actual dimensions differ') + if v['name']=='thumbnail': + check(jpeg_size(b)==dimensions,'thumbnail geometry differs') + if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white') + if v['name']=='crop': + check(actual==(128,128),'crop geometry differs') + if name=='center_crop': + import re + color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b)))) + check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered') + check(b'Exif\0\0' not in b,'EXIF retained in derivative') + c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve) + results.append({'case':name,'result':'PASS','thumbnail':dimensions}) + for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]: + c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime}) + (ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs') +if __name__=='__main__':main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 0000000..6e88ea6 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens.""" +import argparse +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from image_probe import jpeg_size +FAULTS=ROOT/'.lab5-faults' +COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')] +RESULTS=[] +SELECTED=set() + +def compose(*args, allow_failure=False): + p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180) + if p.returncode and not allow_failure: + # Infrastructure logs may contain settings; report only action and exit status. + raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode)) + return p + +def sql(query): + return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip() + +def uid(value): + return str(uuid.UUID(value)) + +def job(att,version=1): + return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version)) + +def operator(action,id=None,version=None, allow_failure=False): + args=['run','--rm','--no-deps','-T','operator','jobs',action] + if id: args+=['--id',uid(id)] + if version:args+=['--pipeline-version',str(version)] + return compose(*args,allow_failure=allow_failure) + +def wait(predicate,timeout=65): + deadline=time.monotonic()+timeout + while time.monotonic()=2,'lease not recovered') + operator('reprocess',att['id'],2);second=ready(c,bob,att,data) + check(first['variants']!=second['variants'],'pipeline versions are indistinguishable') + operator('redeliver',jid);marker.unlink();time.sleep(1) + check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2') + check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back') + ready(c,bob,att,data) + return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8} + record('lease_reorder_and_stale_version',run,version_fence) + def retries(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65) + check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed') + return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6} + record('bounded_retry_and_explicit_replay',run,retries) + def permanent(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid} + record('permanent_error',run,permanent) + for point in ('after_staging','after_upload_commit'): + def api_crash(point=point): + if point=='after_upload_commit':compose('stop','dispatcher') + marker=directive(point,'all','crash') + try:c.attachment(alice,chat) + except Failure:pass + aid=hit(marker);att={'id':aid} + if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists') + else: + check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached') + compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes()) + wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'} + record('api_crash_'+point,run,api_crash) + payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS} + check(bool(RESULTS),'no scenarios selected') + payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab5','--format','{{.Id}}'],text=True).strip() + (ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n') + finally: + for f in FAULTS.iterdir():f.unlink() + compose('stop','worker','dispatcher',allow_failure=True) + # Restore standard configuration, disabling fault injection and short experiment leases. + subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180) + +def health(c): + try:c.call('GET','/health/ready',schema='Health');return True + except Failure:return False + +if __name__=='__main__':main() diff --git a/student-tests/storage.py b/student-tests/storage.py new file mode 100644 index 0000000..e652d8d --- /dev/null +++ b/student-tests/storage.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Staging protection, retirement, retention and explicit restoration on synthetic data.""" +import json +from pathlib import Path +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +BASE=[str(ROOT/'scripts/compose.sh')] +import subprocess + +def command(*args,fail=False): + r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120) + if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0]) + return r + +def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip() +def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail) +def wait(fn): + deadline=time.monotonic()+20 + while time.monotonic()0 and exists(key),'reconciliation removed active source') + results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']}) + command('start','worker');c.wait_ready(bob,att) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='') + check(not exists(key),'ready source staging not retired') + results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']}) + command('stop','worker') + failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id']) + op('storage','remove-object','--id',failed['id']);command('start','worker') + def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed' + wait(is_failed) + check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source') + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png') + check(exists(key),'restored source missing') + # Advance only the synthetic record's age; do not wait a real day or change production retention. + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id']) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='') + check(not exists(key),'expired failed source not retired') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source') + results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25}) + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed) + check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed') + results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid}) + (ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n') + print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified') + finally:command('start','worker') +if __name__=='__main__':main()