forked from mtusi2026/mtusi2026lab5
Реализация на 5
This commit is contained in:
@@ -18,3 +18,6 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
.lab5-faults
|
||||
bin
|
||||
|
||||
+6
-4
@@ -30,9 +30,11 @@ RECONCILE_GRACE_PERIOD=1h
|
||||
# SeaweedFS image and persistent bucket bootstrap are pinned in compose.yaml.
|
||||
# API has four upload slots and one decoder per replica, GOMEMLIMIT=384MiB.
|
||||
|
||||
# Planned lab5 async pipeline settings; the ported lab4 application does not use these yet.
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
# Async lab5 pipeline. Broker credentials are generated by bootstrap.
|
||||
BROKER_PASSWORD=replace-with-local-broker-password
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
JOB_TIMEOUT=60s
|
||||
JOB_LEASE=90s
|
||||
STAGING_RETENTION=24h
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.27.2'
|
||||
cache-dependency-path: service/go.sum
|
||||
- run: make check build vet
|
||||
- run: make up
|
||||
- run: make test test-go test-storage test-images test-staging test-broker-persistence
|
||||
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
- run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
- run: make restore
|
||||
- run: make test-pipeline
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: pipeline-evidence
|
||||
path: evidence/pipeline.json
|
||||
- run: make down
|
||||
if: always()
|
||||
@@ -31,3 +31,5 @@ coverage/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
/.ignore/
|
||||
|
||||
.lab5-faults/
|
||||
|
||||
@@ -16,6 +16,9 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
|
||||
|
||||
FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0
|
||||
|
||||
RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0
|
||||
COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml
|
||||
|
||||
COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
|
||||
@@ -13,8 +13,8 @@ OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen
|
||||
.PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt
|
||||
|
||||
help:
|
||||
@echo "up/down: lab5 stand (ported lab4 application); build/vet: Go checks"
|
||||
@echo "test: lab5 API smoke (requires async implementation); test-go/test-storage: existing Go checks"
|
||||
@echo "up/down: lab5 async image pipeline; build/vet: Go checks"
|
||||
@echo "test: lab5 API smoke; test-go/test-storage: Go checks"
|
||||
@echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic"
|
||||
@echo "check/validate: lab5 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks"
|
||||
@echo "reconcile/reconcile-apply: storage consistency"
|
||||
@@ -111,3 +111,18 @@ reconcile:
|
||||
|
||||
reconcile-apply:
|
||||
./scripts/compose.sh run --rm -T operator storage reconcile --apply
|
||||
|
||||
.PHONY: test-pipeline
|
||||
test-pipeline:
|
||||
$(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json
|
||||
|
||||
.PHONY: test-images test-staging
|
||||
test-images:
|
||||
$(PYTHON) student-tests/images.py
|
||||
|
||||
test-staging:
|
||||
$(PYTHON) student-tests/storage.py
|
||||
|
||||
.PHONY: test-broker-persistence
|
||||
test-broker-persistence:
|
||||
$(PYTHON) student-tests/broker_persistence.py
|
||||
|
||||
@@ -1,37 +1,173 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: не предоставлены владельцем проекта.
|
||||
- Целевая оценка: **5**, трек 2 — доказанная идемпотентность versioned pipeline.
|
||||
- Ревизия: базовый commit и состояние рабочего дерева записываются в `evidence/pipeline.json`; решение пока не закоммичено.
|
||||
- Использованы Go, pgx, AWS SDK, RabbitMQ `amqp091-go`, ImageMagick;
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Compose и для локальных Go-проверок Go 1.27+. Контейнерная сборка использует Go 1.27.2. Образы закреплены версиями/digest в `compose.yaml` и `Dockerfile`; ImageMagick и модули JPEG/WebP — 7.1.2.35-r0.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
make check build vet
|
||||
make up
|
||||
curl --fail http://localhost:18085/health/ready
|
||||
make test
|
||||
make test-go
|
||||
make test-storage
|
||||
make test-images test-staging test-broker-persistence
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
make restore
|
||||
make test-pipeline
|
||||
make down
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Полная валидация схем:
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
Bootstrap сохраняет существующие локальные настройки и генерирует синтетические credentials PostgreSQL, Redis, S3 и RabbitMQ. `.env.lab5` и `secrets/lab5` игнорируются Git. Миграции выполняются отдельным одноразовым процессом, bucket создаётся идемпотентно. Внешний endpoint — только `127.0.0.1:18085`; внутренние хранилища портов на host не публикуют. API-реплик по умолчанию две.
|
||||
|
||||
`make down` сохраняет volumes. **Намеренное удаление учебных данных**: `./scripts/compose.sh down --volumes`; эта команда не входит в обычные проверки и автоматически не выполняется.
|
||||
|
||||
## Архитектура и границы
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client --> LB[HAProxy]
|
||||
LB --> API[API x2]
|
||||
API --> Staging[Private S3 staging]
|
||||
API --> DB[PostgreSQL: metadata + jobs + outbox]
|
||||
DB --> Dispatcher
|
||||
Dispatcher --> Broker[RabbitMQ durable queue]
|
||||
Broker --> Worker
|
||||
Worker --> Results[Private S3 versioned results]
|
||||
Worker --> DB
|
||||
```
|
||||
|
||||
Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Attachment service принимает DTO и reader, проверяет доступ и целостность изображения, управляет транзакцией и состояниями. Репозитории выполняют операции хранения; HTTP, retry-политик и преобразования изображений в SQL/триггерах нет. Канонизация тела сообщения, сравнение идемпотентных запросов и создание Message перенесены в chat service.
|
||||
|
||||
До `202` весь проверенный оригинал уже записан в S3 staging, а metadata, job и outbox — в одной завершённой транзакции PostgreSQL. При неопределённом COMMIT нет слепого компенсирующего DELETE. Ответ содержит сохранённый снимок `queued`, даже если быстрый worker уже начал работу.
|
||||
|
||||
Staging находится в том же S3-сервисе, что конечные объекты: это разделение prefix и жизненных циклов, а не обещание отложить всю сетевую передачу файла. Проверка корректности изображения при upload включает ограниченное декодирование; resize/crop/WebP и публикация конечного оригинала выполняются отдельно в worker.
|
||||
|
||||
API readiness проверяет PostgreSQL и S3. Недоступность broker/worker не блокирует транзакционный приём через outbox. Worker и dispatcher переподключаются после отказа зависимостей. Redis остаётся необязательным кэшем, очередь в него не помещается.
|
||||
|
||||
### Инфраструктура
|
||||
|
||||
| Роль → service | Версия | Проверка | Volume / сеть / credentials |
|
||||
| --- | --- | --- | --- |
|
||||
| API → api, ≥2 | собственная сборка Go 1.27.2 | `/health/ready` | без локального durable state; default + storage; signing/S3/broker files, DB env |
|
||||
| PostgreSQL → postgres | 17.11, digest в Compose | pg_isready | postgres_data; default; локальный пароль env |
|
||||
| Кэш → redis | 8.2.11, digest в Compose | authenticated PING | redis_data, AOF; default; локальный пароль env |
|
||||
| LB → lb | HAProxy 3.2, digest в Compose | backend readiness | без volume данных; default; публичный localhost порт |
|
||||
| S3 → s3 | SeaweedFS 4.52, digest в Compose | cluster status; bootstrap проверяет запись | s3_data; internal storage; credentials JSON file |
|
||||
| Broker → broker | RabbitMQ 4.2.0, digest в Compose | rabbitmq-diagnostics check_running | broker_data; default; синтетические credentials |
|
||||
| Worker → worker | образ приложения + ImageMagick | процесс/reconnect, состояние jobs | эфемерные ограниченные temp files; default + storage; secret files |
|
||||
| Dispatcher → dispatcher | образ приложения | процесс/reconnect, состояние outbox | без локального durable state; default; broker secret file |
|
||||
|
||||
У worker/dispatcher отключён унаследованный HTTP healthcheck: HTTP-сервер они не запускают. Состояние `running` само по себе не подтверждает прогресс; проверять нужно jobs/outbox и end-to-end сценарий.
|
||||
|
||||
### Доставка и повторы
|
||||
|
||||
- Durable RabbitMQ queue, persistent messages, publisher confirms и mandatory routing. Имя broker фиксировано, чтобы новый контейнер открывал ту же persistent node database; readiness ждёт запуска RabbitMQ application, а не только Erlang node. Consumer ack выполняется после устойчивого результата, постановки следующей попытки либо записи terminal failed.
|
||||
- Сообщения соответствуют `jobs.schema.json`: UUID, версия, attempt, timestamp, опциональный trace carrier. Байтов файла, URL и credentials в них нет. Worker читает доверенные storage metadata из PostgreSQL.
|
||||
- `job_id` стабилен для вложения и версии. Redelivery сама не увеличивает attempt; управляемый retry/recovery увеличивает его. По умолчанию пять попыток, exponential backoff с jitter 0.5–30 секунд в зависимости от номера попытки.
|
||||
- Publisher имеет 20-секундный deadline на самом socket (одного контекста PublishWithContext недостаточно для ограничения сетевой записи). Dispatcher захватывает одну запись с lease 30 секунд, публикует её и фиксирует результат с проверкой token. Просроченный dispatcher не может изменить запись нового владельца. После пяти неудачных публикаций остаётся dead_letter-запись и наблюдаемый failed job.
|
||||
- Worker: concurrency=2, prefetch=2, timeout=60s, lease=90s. Lease recovery работает независимо от consumer. Неактивный lease не продлевается: работа дольше лимита повторяется ограниченно.
|
||||
- Постоянная ошибка заканчивается немедленно (`invalid_image`, `source_missing`, `source_corrupt`); временная повторяется и после исчерпания бюджета даёт `retry_exhausted`. API получает только безопасный error_code.
|
||||
- Реестр failed — PostgreSQL. Replay явный, только для текущего failed job с доступным source; записывается аудит, номер attempt продолжает расти, выдаётся новый бюджет из пяти попыток. Уже успешную job replay не запускает.
|
||||
|
||||
```sh
|
||||
./scripts/compose.sh run --rm -T operator jobs list
|
||||
./scripts/compose.sh run --rm -T operator jobs replay --id JOB_UUID
|
||||
./scripts/compose.sh run --rm -T operator outbox replay --id JOB_UUID
|
||||
./scripts/compose.sh run --rm -T operator jobs reprocess --id ATTACHMENT_UUID --pipeline-version 2
|
||||
```
|
||||
|
||||
`outbox replay` восстанавливает логическую failed job вместе с новой outbox-записью; ID аргумента — job, не отдельная доставка. Старые dead_letter-записи сохраняются для диагностики и не переводят новую попытку обратно в failed.
|
||||
|
||||
### Версии и инварианты
|
||||
|
||||
Manifest имеет PK `(attachment_id, pipeline_version, variant)`. Варианты публикуются одной DB-транзакцией только после записи всех объектов. Завершение проверяет текущую версию вложения, состояние job, token и действующий lease. Ready не откатывается из-за старой доставки.
|
||||
|
||||
Объектные ключи включают вложение, версию, variant и SHA-256. Повтор одинакового эффекта использует тот же ключ; другая версия пишет в свой namespace. Поэтому устаревшая попытка не перезапишет новые байты S3, даже когда БД отклоняет её завершение. Незавершённые объекты возможны и удаляются как сироты после grace; они не становятся опубликованным вторым набором вариантов.
|
||||
|
||||
Версия 1 использует WebP quality 80, версия 2 — 85. Остальные параметры одинаковы. Reprocess доступен для ready-вложения и строго большей поддерживаемой версии; на время новой работы статус queued и content=409. Сохраняются прежние manifest, новый оригинал побайтно совпадает с исходником.
|
||||
|
||||
### Изображения и cleanup
|
||||
|
||||
Thumbnail: JPEG, вписан в 256×256 без увеличения, размеры округляются вниз с минимумом 1 px. Crop: центральный квадрат JPEG 128×128 с разрешённым увеличением. Optimized: WebP внутри 1280×1280 без увеличения. Применяется EXIF orientation, alpha компонуется на белый, метаданные производных удаляются. JPEG quality=85. Не обещается уменьшение каждого файла.
|
||||
|
||||
ImageMagick запускается без shell, отдельным дочерним процессом с отменой по timeout. Политика запрещает delegates/filters и разрешает только нужные coders; memory=256MiB, map=0, disk=256MiB, thread=1, time=60s, list-length=4. Worker ограничен 1 CPU/1GiB; его локальные файлы — только восстанавливаемые временные данные. API принимает максимум четыре одновременных upload и выполняет одну проверку декодирования на реплику. Пиксели проверяются до полной распаковки. Превышение ресурсов заканчивается ошибкой, а не бесконечным retry.
|
||||
|
||||
Staging удаляется после устойчивого ready; failed source хранится 24h. Источники, которыми являются исторические опубликованные results, физически сохраняются. После удаления failed source replay отклоняется; оператор может восстановить оригинал с проверкой размера/SHA и затем явно повторить job. Reconcile защищает ссылки metadata/manifest и активные jobs, удаляет сироты только после grace=1h (>2×upload timeout). Старые originals lab4 переводятся миграцией на первую версию pipeline без изменения ID и ссылок сообщений.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Фактический вывод команд сохраняется отдельно от описания реализации. `evidence/pipeline.json` содержит повторяемые сценарии, UUID, длительности и результат каждого запуска; `evidence/images.json` — проверки изображений. Проверки защиты staging, удаления после ready, retention failed и восстановления исходника — `evidence/storage.json`. Остальные результаты — `evidence/checks.txt`.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий | Команда / наблюдение | Доказательство |
|
||||
| --- | --- | --- |
|
||||
| Структура, схемы, сборка и анализ | make check; make validate; make build; make vet | checks.txt |
|
||||
| Публичный контракт и предыдущие лабораторные | make test; make test-go; make test-storage | checks.txt |
|
||||
| Сессия и данные после перезапуска; отказ API | test-persistence / test-failover с действительными hooks | checks.txt |
|
||||
| 202 при остановленном worker, сообщение с queued-вложением, рестарт stores/broker и ready | worker_down_and_store_restart | pipeline.json |
|
||||
| Опубликованное сообщение переживает пересоздание broker с тем же volume при остановленном dispatcher | make test-broker-persistence; hostname=broker сохраняет имя узла | broker-persistence.json |
|
||||
| Thumbnail/crop/WebP, orientation, alpha, реальные размеры и hashes | student-tests/images.py | images.json |
|
||||
| Сбой до/после S3 side effect и до/после COMMIT результата, до ack | crash_before_objects / after_original_put / after_objects / after_result_commit / before_ack | pipeline.json |
|
||||
| Confirm получен, outbox ещё processing; повтор публикации безопасен | crash_confirm_before_outbox_mark | pipeline.json |
|
||||
| Ready redelivery — no-op | ready_redelivery, 3 копии | pipeline.json |
|
||||
| Реальный lease expiry, более новая попытка/версия, возобновление старого worker и поздняя старая доставка | lease_reorder_and_stale_version | pipeline.json |
|
||||
| Staging защищён при queued; после ready удалён; failed retention и restore проверены | make test-staging | storage.json |
|
||||
| Retry ограничен, failed наблюдаем, replay явный | bounded_retry_and_explicit_replay / permanent_error | pipeline.json |
|
||||
| Сбой после staging и разрыв COMMIT/publish | api_crash_after_staging / api_crash_after_upload_commit | pipeline.json |
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Эксперимент трека 2
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Гипотеза:** атомарное сохранение намерения, durable доставка, fencing в БД и разделение версий/контента в S3 сохраняют принятые задания и предотвращают публикацию устаревшего эффекта при проверяемых crash/reorder/duplicate сценариях.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Методика: `make test-pipeline` включает исключительно для эксперимента `compose.faults.yaml`. Директивы в игнорируемом `.lab5-faults/` адресуются по job/attachment ID и точке сбоя. Барьер отмечает достижение точки; crash завершает процесс с кодом 86, restart policy поднимает его снова. Pause применяется один раз: другая попытка может завершиться после реального истечения lease. Общая директория не содержит токенов и очищается при завершении.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
В эксперименте timeout=5s, lease=8s, concurrency/prefetch=2; остальные параметры не ускоряются искусственным изменением lease в БД. Окно после publisher confirm использует обычный outbox lease=30s. Для гарантированного повторного publish тест повторно делает свою outbox-запись pending; это внесение дубля, а не штатная операция пользователя. Для COMMIT/publish dispatcher заранее остановлен, проверяется pending-запись. Для crash до COMMIT отсутствие `202` не трактуется как потеря принятой работы.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Каждый сценарий выполняется три раза с новыми синтетическими пользователями/вложениями. Основной файл — публичный PNG fixture 320×200, 111817 bytes; дополнительных параллельных upload в fault-эксперименте нет. Измеряется длительность сценария (включает Docker stop/start и проверки), а не чистая latency обработки. Точность таймеров — monotonic clock клиента; нагрузочный максимум этим экспериментом не измеряется.
|
||||
|
||||
Окружение выполнения: Apple M4 Pro, 14 CPU, host RAM 24GiB; Docker Desktop arm64, 14 CPU и около 7.75GiB RAM. CPU/RAM контейнеров ограничены Compose. Исследование проводится на одном Docker-host с сохранением volumes, без симуляции потери диска.
|
||||
|
||||
|
||||
### Наблюдавшиеся результаты
|
||||
|
||||
Все **39/39** сценариев трека завершились PASS (13 сценариев × 3 повтора). Интервалы ниже включают действия Docker и проверки; это не измерение чистого processing latency.
|
||||
|
||||
| Сценарий | Минимум–максимум, с |
|
||||
| --- | --- |
|
||||
| worker_down_and_store_restart | 63.864–66.563 |
|
||||
| crash_before_objects | 10.888–12.269 |
|
||||
| crash_after_original_put | 11.222–12.171 |
|
||||
| crash_after_objects | 11.174–11.340 |
|
||||
| crash_after_result_commit | 1.973–2.088 |
|
||||
| crash_before_ack | 2.017–2.098 |
|
||||
| ready_redelivery | 2.435–3.002 |
|
||||
| crash_confirm_before_outbox_mark | 33.988–36.021 |
|
||||
| lease_reorder_and_stale_version | 14.652–15.225 |
|
||||
| bounded_retry_and_explicit_replay | 13.737–15.238 |
|
||||
| permanent_error | 2.552–2.699 |
|
||||
| api_crash_after_staging | 0.144–0.817 |
|
||||
| api_crash_after_upload_commit | 2.141–2.328 |
|
||||
|
||||
После эксперимента отдельная проверка retention обнаружила и позволила исправить гонку восстановления исходника: Restore обновляет возраст source, а Cleanup берёт тот же advisory lock. Эта корректировка проверена `make test-staging`. После добавления socket deadline повторно выполнены confirm/crash, lease + stale version и COMMIT/publish на итоговом образе: **3/3 PASS**, см. `evidence/pipeline-final.json`. Image ID исходного полного fault-прогона сохранён в pipeline.json, итогового — в pipeline-final.json.
|
||||
|
||||
## Ограничения
|
||||
|
||||
At-least-once доставка сохраняется: publish/confirm/COMMIT/ack не образуют одну распределённую транзакцию. Доказывается идемпотентность опубликованного эффекта в перечисленных сценариях, а не exactly-once доставка и не отсутствие любых возможных отказов.
|
||||
|
||||
Один PostgreSQL, S3, broker, LB и host — отдельные точки отказа. Durable очередь здесь не означает кластер RabbitMQ. Невосстановимая потеря volumes не покрывается; backup/restore инфраструктуры не реализован. После окончательной ошибки нужен операторский replay; при длительном отказе брокера принятые задания могут стать наблюдаемыми failed. Накопление outbox/jobs не ограничено глобальной квотой: ограничены upload, worker concurrency/prefetch и ресурсы, но отдельный трек перегрузки не заявляется.
|
||||
|
||||
Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acks](https://www.rabbitmq.com/docs/confirms), [Go client](https://github.com/rabbitmq/amqp091-go), [ImageMagick resource policy](https://imagemagick.org/resources/).
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# Explicit isolated experiment profile. Never enabled by make up.
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
LAB5_FAULTS: "1"
|
||||
volumes:
|
||||
- ./.lab5-faults:/faults
|
||||
worker:
|
||||
environment:
|
||||
LAB5_FAULTS: "1"
|
||||
JOB_TIMEOUT: 5s
|
||||
JOB_LEASE: 8s
|
||||
volumes:
|
||||
- ./.lab5-faults:/faults
|
||||
dispatcher:
|
||||
environment:
|
||||
LAB5_FAULTS: "1"
|
||||
volumes:
|
||||
- ./.lab5-faults:/faults
|
||||
operator:
|
||||
environment:
|
||||
LAB5_FAULTS: "1"
|
||||
@@ -40,6 +40,13 @@ x-api: &api
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
|
||||
JOB_LEASE: ${JOB_LEASE:-90s}
|
||||
STAGING_RETENTION: ${STAGING_RETENTION:-24h}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s}
|
||||
@@ -47,6 +54,7 @@ x-api: &api
|
||||
GOMEMLIMIT: 384MiB
|
||||
networks: [default, storage]
|
||||
volumes:
|
||||
- ./secrets/lab5/broker_url:/run/secrets/broker_url:ro
|
||||
- ./secrets/lab5/private_key.pem:/run/certs/private_key.pem:ro
|
||||
- ./secrets/lab5/public_key.pem:/run/certs/public_key.pem:ro
|
||||
- ./secrets/lab5/s3_api_access:/run/secrets/s3_access:ro
|
||||
@@ -78,6 +86,50 @@ services:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
hostname: broker
|
||||
image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh}
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: [CMD, rabbitmq-diagnostics, -q, check_running]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
stop_grace_period: 30s
|
||||
worker:
|
||||
<<: *api
|
||||
command: [worker]
|
||||
cpus: 1
|
||||
mem_limit: 1g
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
dispatcher:
|
||||
<<: *api
|
||||
command: [dispatcher]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
lb:
|
||||
image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0}
|
||||
restart: unless-stopped
|
||||
@@ -168,6 +220,7 @@ services:
|
||||
- ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro
|
||||
- ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro
|
||||
- ./tests/fixtures:/fixtures:ro
|
||||
- ./secrets/lab5/broker_url:/run/secrets/broker_url:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on: {}
|
||||
@@ -176,6 +229,7 @@ volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
broker_data:
|
||||
|
||||
networks:
|
||||
storage:
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"attachment_id": "cb1e8982-980d-4286-8224-335b644a5fc5",
|
||||
"broker_hostname": "broker",
|
||||
"old_container_id": "16848707eae7bdfb195da43c912ad66399678cc0a48d100bea119619f7e62009",
|
||||
"new_container_id": "0dfeb1f83d39adc4de9c7798d65b5f125927fe95ae050e4a17e8a5238713b81b",
|
||||
"message_present_before_and_after": true,
|
||||
"outbox_published_before_replacement": true,
|
||||
"dispatcher_stopped_during_replacement": true
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"date_utc": "2026-11-13T09:29:53.527687+00:00",
|
||||
"base_commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
|
||||
"working_tree": "uncommitted solution",
|
||||
"runtime_source_sha256": "cadfe8fc0d834a598d26e03d1c0348c907ac6fe9a4574aa959f156dfd69488d6",
|
||||
"hash_method": "Sorted relative path + NUL + file content + NUL, for service/**/*.go, migrations/*.sql, go.mod, go.sum, Dockerfile and ImageMagick policy",
|
||||
"image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f",
|
||||
"go_local": "go version go1.27.0 darwin/arm64",
|
||||
"docker_server": "29.7.2",
|
||||
"compose": "Docker Compose version v5.3.1",
|
||||
"docker_resources": "cpus=14 memory_bytes=8320954368 arch=aarch64",
|
||||
"hardware": "Apple M4 Pro, 14 CPUs, 24 GiB host RAM",
|
||||
"source_reuse": "Adapted core outbox from supplied mamont-optics project; original project left unchanged"
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
Lab5 verification, UTC 2026-11-13T09:29:53.527687+00:00
|
||||
All commands below completed with exit code 0.
|
||||
Go output excerpts omit SQL, runtime logs and synthetic request bodies.
|
||||
|
||||
$ make check build vet; make validate PYTHON=.venv/bin/python
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
cd service && go build ./...
|
||||
cd service && go build -o ../bin/benchmark ./cmd/benchmark
|
||||
cd service && go vet ./...
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://localhost:18085"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
$ make test-go (race enabled)
|
||||
? gitea.privcloud.tech/mtusi2026/cmd/app [no test files]
|
||||
=== RUN TestLostPostResponseIsNotRetried
|
||||
--- PASS: TestLostPostResponseIsNotRetried (0.00s)
|
||||
PASS
|
||||
ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.461s
|
||||
? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files]
|
||||
=== RUN TestIdempotencyTransactionAndExpiry
|
||||
PASS
|
||||
ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.653s
|
||||
? gitea.privcloud.tech/mtusi2026/internal/config [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/dto [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files]
|
||||
=== RUN TestSessionFingerprints
|
||||
--- PASS: TestSessionFingerprints (0.02s)
|
||||
PASS
|
||||
ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.908s
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files]
|
||||
=== RUN TestVerifiedSessionIsRequestScoped
|
||||
--- PASS: TestVerifiedSessionIsRequestScoped (0.00s)
|
||||
=== RUN TestMissingTokenDoesNotCallSessionStore
|
||||
--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s)
|
||||
PASS
|
||||
ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 2.467s
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files]
|
||||
? gitea.privcloud.tech/mtusi2026/migrations [no test files]
|
||||
=== RUN TestInvalidJSON
|
||||
=== RUN TestInvalidJSON/malformed
|
||||
=== RUN TestInvalidJSON/wrong_type
|
||||
=== RUN TestInvalidJSON/unknown_field
|
||||
--- PASS: TestInvalidJSON (0.01s)
|
||||
--- PASS: TestInvalidJSON/malformed (0.01s)
|
||||
--- PASS: TestInvalidJSON/wrong_type (0.00s)
|
||||
--- PASS: TestInvalidJSON/unknown_field (0.00s)
|
||||
=== RUN TestForeignChatIsHidden
|
||||
--- PASS: TestForeignChatIsHidden (0.54s)
|
||||
=== RUN TestEmptyMessage
|
||||
--- PASS: TestEmptyMessage (0.21s)
|
||||
=== RUN TestDuplicateUsername
|
||||
--- PASS: TestDuplicateUsername (0.28s)
|
||||
=== RUN TestMessagePagination
|
||||
--- PASS: TestMessagePagination (0.22s)
|
||||
=== RUN TestAttachmentContractAndErrors
|
||||
=== RUN TestAttachmentContractAndErrors/fake
|
||||
=== RUN TestAttachmentContractAndErrors/mime_mismatch
|
||||
=== RUN TestAttachmentContractAndErrors/wrong_type
|
||||
=== RUN TestAttachmentContractAndErrors/truncated
|
||||
--- PASS: TestAttachmentContractAndErrors (1.05s)
|
||||
--- PASS: TestAttachmentContractAndErrors/fake (0.00s)
|
||||
--- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s)
|
||||
--- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s)
|
||||
--- PASS: TestAttachmentContractAndErrors/truncated (0.00s)
|
||||
=== RUN TestConcurrentAttachmentMessage
|
||||
--- PASS: TestConcurrentAttachmentMessage (0.25s)
|
||||
=== RUN TestConcurrentMessages
|
||||
--- PASS: TestConcurrentMessages (1.84s)
|
||||
=== RUN TestIdempotencyContract
|
||||
--- PASS: TestIdempotencyContract (0.45s)
|
||||
=== RUN TestConcurrentIdempotencyAcrossReplicas
|
||||
--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s)
|
||||
PASS
|
||||
ok gitea.privcloud.tech/mtusi2026/tests 7.261s
|
||||
|
||||
$ make test-storage (isolated database)
|
||||
=== RUN TestIdempotencyTransactionAndExpiry
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object
|
||||
=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry (0.12s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.02s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s)
|
||||
--- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s)
|
||||
PASS
|
||||
|
||||
$ make test-images
|
||||
PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs
|
||||
|
||||
$ make test-staging
|
||||
PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified
|
||||
|
||||
$ make test-broker-persistence
|
||||
PASS: confirmed message survived broker container replacement with the same volume and stable node identity
|
||||
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.273}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
Actual process state before persistence hook
|
||||
/mtusi2026lab5-api-1 pid=63585 started=2026-11-13T09:13:37.287792805Z
|
||||
/mtusi2026lab5-api-2 pid=63443 started=2026-11-13T09:13:37.185017014Z
|
||||
/mtusi2026lab5-broker-1 pid=16156 started=2026-11-13T09:06:10.180568751Z
|
||||
/mtusi2026lab5-dispatcher-1 pid=63444 started=2026-11-13T09:13:37.187137055Z
|
||||
/mtusi2026lab5-lb-1 pid=6588 started=2026-11-13T08:49:29.974451052Z
|
||||
/mtusi2026lab5-postgres-1 pid=15734 started=2026-11-13T09:06:09.241419334Z
|
||||
/mtusi2026lab5-redis-1 pid=15771 started=2026-11-13T09:06:09.289178667Z
|
||||
/mtusi2026lab5-s3-1 pid=17699 started=2026-11-13T09:06:34.353438762Z
|
||||
/mtusi2026lab5-worker-1 pid=66553 started=2026-11-13T09:14:11.339788001Z
|
||||
|
||||
Actual process state after persistence hook
|
||||
/mtusi2026lab5-api-1 pid=73635 started=2026-11-13T09:15:52.601737132Z
|
||||
/mtusi2026lab5-api-2 pid=73510 started=2026-11-13T09:15:52.490649465Z
|
||||
/mtusi2026lab5-broker-1 pid=70234 started=2026-11-13T09:14:51.740755173Z
|
||||
/mtusi2026lab5-dispatcher-1 pid=73512 started=2026-11-13T09:15:52.489731548Z
|
||||
/mtusi2026lab5-lb-1 pid=73700 started=2026-11-13T09:15:52.696327173Z
|
||||
/mtusi2026lab5-postgres-1 pid=70113 started=2026-11-13T09:14:50.775761506Z
|
||||
/mtusi2026lab5-redis-1 pid=70044 started=2026-11-13T09:14:50.718397506Z
|
||||
/mtusi2026lab5-s3-1 pid=71476 started=2026-11-13T09:15:16.065232545Z
|
||||
/mtusi2026lab5-worker-1 pid=73511 started=2026-11-13T09:15:52.490188673Z
|
||||
|
||||
Actual state after API failover, before restoring stopped replica
|
||||
api mtusi2026lab5-api-1 running healthy
|
||||
api mtusi2026lab5-api-2 exited
|
||||
broker mtusi2026lab5-broker-1 running healthy
|
||||
dispatcher mtusi2026lab5-dispatcher-1 running
|
||||
lb mtusi2026lab5-lb-1 running healthy
|
||||
migrate mtusi2026lab5-migrate-1 exited
|
||||
postgres mtusi2026lab5-postgres-1 running healthy
|
||||
redis mtusi2026lab5-redis-1 running healthy
|
||||
s3 mtusi2026lab5-s3-1 running healthy
|
||||
s3-init mtusi2026lab5-s3-init-1 exited
|
||||
worker mtusi2026lab5-worker-1 running
|
||||
|
||||
Effective container resource limits
|
||||
/mtusi2026lab5-api-1 memory_bytes=536870912 cpu_nano=1000000000
|
||||
/mtusi2026lab5-api-2 memory_bytes=536870912 cpu_nano=1000000000
|
||||
/mtusi2026lab5-broker-1 memory_bytes=536870912 cpu_nano=500000000
|
||||
/mtusi2026lab5-dispatcher-1 memory_bytes=134217728 cpu_nano=250000000
|
||||
/mtusi2026lab5-lb-1 memory_bytes=134217728 cpu_nano=500000000
|
||||
/mtusi2026lab5-postgres-1 memory_bytes=1073741824 cpu_nano=2000000000
|
||||
/mtusi2026lab5-redis-1 memory_bytes=134217728 cpu_nano=250000000
|
||||
/mtusi2026lab5-s3-1 memory_bytes=536870912 cpu_nano=1000000000
|
||||
/mtusi2026lab5-worker-1 memory_bytes=1073741824 cpu_nano=1000000000
|
||||
|
||||
make test-pipeline: 39/39 PASS; details and repetitions in pipeline.json
|
||||
Final image targeted fault checks: 3/3 PASS; details in pipeline-final.json
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"case": "small",
|
||||
"result": "PASS",
|
||||
"thumbnail": [
|
||||
16,
|
||||
8
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "floor",
|
||||
"result": "PASS",
|
||||
"thumbnail": [
|
||||
256,
|
||||
76
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "alpha",
|
||||
"result": "PASS",
|
||||
"thumbnail": [
|
||||
40,
|
||||
20
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "orientation",
|
||||
"result": "PASS",
|
||||
"thumbnail": [
|
||||
128,
|
||||
256
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "thin",
|
||||
"result": "PASS",
|
||||
"thumbnail": [
|
||||
1,
|
||||
256
|
||||
]
|
||||
},
|
||||
{
|
||||
"case": "center_crop",
|
||||
"result": "PASS",
|
||||
"thumbnail": [
|
||||
256,
|
||||
85
|
||||
]
|
||||
}
|
||||
],
|
||||
"negative_cases": 4
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"date_utc": "2026-11-13T09:24:08.989020+00:00",
|
||||
"commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
|
||||
"working_tree": "implementation changes uncommitted",
|
||||
"repeats": 1,
|
||||
"worker": {
|
||||
"concurrency": 2,
|
||||
"prefetch": 2,
|
||||
"timeout_seconds": 5,
|
||||
"lease_seconds": 8,
|
||||
"memory": "1GiB",
|
||||
"cpus": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"scenario": "crash_confirm_before_outbox_mark",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 33.527,
|
||||
"job_id": "ce8a7658-c069-429a-b655-cb051f922e14"
|
||||
},
|
||||
{
|
||||
"scenario": "lease_reorder_and_stale_version",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 14.623,
|
||||
"attachment_id": "6e709809-9ca2-4fb8-b7f6-155a3aace89c",
|
||||
"old_job_id": "421fa4dc-3704-4f87-a490-3110f062459a",
|
||||
"pipeline_version": 2,
|
||||
"manifest_rows": 8
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_upload_commit",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 2.122,
|
||||
"attachment_id": "293e92d6-4253-4fd9-8bad-719a488f6bdf",
|
||||
"accepted_commit": true
|
||||
}
|
||||
],
|
||||
"image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f"
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
{
|
||||
"date_utc": "2026-11-13T09:09:05.756165+00:00",
|
||||
"commit": "ce035c85282e75513ea36cb429f4b8d775398d62",
|
||||
"working_tree": "implementation changes uncommitted",
|
||||
"repeats": 3,
|
||||
"worker": {
|
||||
"concurrency": 2,
|
||||
"prefetch": 2,
|
||||
"timeout_seconds": 5,
|
||||
"lease_seconds": 8,
|
||||
"memory": "1GiB",
|
||||
"cpus": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"scenario": "worker_down_and_store_restart",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 66.563,
|
||||
"attachment_id": "912b15f8-2ddb-4cbc-8db7-526540256749",
|
||||
"job_id": "4913bc34-1533-4140-a0dc-c53efa5be0dd",
|
||||
"postgres_restarted": true
|
||||
},
|
||||
{
|
||||
"scenario": "crash_before_objects",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 10.888,
|
||||
"job_id": "bfdb496b-f2e0-4836-9559-1a9b07ed21c4",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_original_put",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 12.171,
|
||||
"job_id": "8468e628-5eef-42c2-9a86-d050d74cb672",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_objects",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 11.174,
|
||||
"job_id": "4c587b0f-fa79-4d9e-a3c7-4904db1e80d3",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_result_commit",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 1.973,
|
||||
"job_id": "419c9eeb-6467-4138-b05e-92f1bc929df8",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_before_ack",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 2.017,
|
||||
"job_id": "face4c1b-5597-4b24-8f61-a0e756e601de",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "ready_redelivery",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 2.435,
|
||||
"job_id": "514c616d-367c-4693-8d33-60079b2ced87",
|
||||
"copies": 3
|
||||
},
|
||||
{
|
||||
"scenario": "crash_confirm_before_outbox_mark",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 33.996,
|
||||
"job_id": "71f7c962-009e-4982-ae7b-e7f24fdcdc7f"
|
||||
},
|
||||
{
|
||||
"scenario": "lease_reorder_and_stale_version",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 14.652,
|
||||
"attachment_id": "52a2e241-60a1-4443-8ef7-2e8e3e860577",
|
||||
"old_job_id": "1c5824ca-774e-4230-9764-3e0938f4efeb",
|
||||
"pipeline_version": 2,
|
||||
"manifest_rows": 8
|
||||
},
|
||||
{
|
||||
"scenario": "bounded_retry_and_explicit_replay",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 14.47,
|
||||
"job_id": "8b8f8975-f4c8-45cf-80e9-00626c4e5c36",
|
||||
"failed_attempt": 5,
|
||||
"replayed_attempt": 6
|
||||
},
|
||||
{
|
||||
"scenario": "permanent_error",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 2.699,
|
||||
"job_id": "e1948ce8-b194-4f66-8008-6e733ad42959"
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_staging",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 0.817,
|
||||
"attachment_id": "2d831ec0-5918-4290-8e26-3c2bec3cc0fd",
|
||||
"accepted_commit": false
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_upload_commit",
|
||||
"run": 1,
|
||||
"result": "PASS",
|
||||
"seconds": 2.328,
|
||||
"attachment_id": "f0da85c6-003c-4d79-ba01-f3f961064741",
|
||||
"accepted_commit": true
|
||||
},
|
||||
{
|
||||
"scenario": "worker_down_and_store_restart",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 66.013,
|
||||
"attachment_id": "64517fe4-6419-4257-bbf5-a47a88b51068",
|
||||
"job_id": "873b9510-7ada-4dc0-bdfb-3e204ae2bc3d",
|
||||
"postgres_restarted": true
|
||||
},
|
||||
{
|
||||
"scenario": "crash_before_objects",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 11.241,
|
||||
"job_id": "1c0ea968-87d2-45df-bb9b-d8b45c68dc1d",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_original_put",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 11.222,
|
||||
"job_id": "748017a5-8f09-4741-8e35-35e4dbcb3c55",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_objects",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 11.34,
|
||||
"job_id": "489d2edd-b008-48fc-ba35-c9f64ee05ba2",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_result_commit",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 2.088,
|
||||
"job_id": "0ed57d8a-3fcc-42a4-a31f-2f02ae4f5f5f",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_before_ack",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 2.026,
|
||||
"job_id": "e9a8270c-3132-460a-a888-ee8722798e35",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "ready_redelivery",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 2.704,
|
||||
"job_id": "b4f7d69b-3de5-4350-97bf-0e880ed94b0e",
|
||||
"copies": 3
|
||||
},
|
||||
{
|
||||
"scenario": "crash_confirm_before_outbox_mark",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 33.988,
|
||||
"job_id": "5f7a89af-a045-423c-8c70-47b19615b577"
|
||||
},
|
||||
{
|
||||
"scenario": "lease_reorder_and_stale_version",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 14.694,
|
||||
"attachment_id": "a1230ca9-8001-4bb4-b648-51446f646cbf",
|
||||
"old_job_id": "2f1e1fbf-856d-4605-873c-b23011072a4e",
|
||||
"pipeline_version": 2,
|
||||
"manifest_rows": 8
|
||||
},
|
||||
{
|
||||
"scenario": "bounded_retry_and_explicit_replay",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 13.737,
|
||||
"job_id": "737209e2-5d3c-4693-8394-fc8173b88ee2",
|
||||
"failed_attempt": 5,
|
||||
"replayed_attempt": 6
|
||||
},
|
||||
{
|
||||
"scenario": "permanent_error",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 2.552,
|
||||
"job_id": "e5e6dc90-9ea5-4d7c-aa11-c6c275d69793"
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_staging",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 0.153,
|
||||
"attachment_id": "ed8e2936-47fa-4717-9eb3-e63292c8a0ad",
|
||||
"accepted_commit": false
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_upload_commit",
|
||||
"run": 2,
|
||||
"result": "PASS",
|
||||
"seconds": 2.159,
|
||||
"attachment_id": "a739f237-2278-421c-aa55-6024e717ad89",
|
||||
"accepted_commit": true
|
||||
},
|
||||
{
|
||||
"scenario": "worker_down_and_store_restart",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 63.864,
|
||||
"attachment_id": "95176e6d-301a-4725-9861-7133507184eb",
|
||||
"job_id": "e5f8b9fa-e460-4441-92b0-2af2ac2540bf",
|
||||
"postgres_restarted": true
|
||||
},
|
||||
{
|
||||
"scenario": "crash_before_objects",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 12.269,
|
||||
"job_id": "67c70b6e-ea42-4c1d-894c-d2234a70bf03",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_original_put",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 11.402,
|
||||
"job_id": "386f9b0d-1b1f-42b6-95a0-a13d6b97f118",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_objects",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 11.259,
|
||||
"job_id": "9c639085-f040-4989-90ba-54376c64630d",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_after_result_commit",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 2.068,
|
||||
"job_id": "73fe51b1-32eb-4c93-b8b0-0ed920a6641d",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "crash_before_ack",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 2.098,
|
||||
"job_id": "2947da70-b658-48c0-9aa2-905bc0927781",
|
||||
"manifest_rows": 4
|
||||
},
|
||||
{
|
||||
"scenario": "ready_redelivery",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 3.002,
|
||||
"job_id": "4f67af5a-1014-4f91-9a68-54739f2edde7",
|
||||
"copies": 3
|
||||
},
|
||||
{
|
||||
"scenario": "crash_confirm_before_outbox_mark",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 36.021,
|
||||
"job_id": "c4135c71-64e7-44ba-9e4f-bf8f7489a459"
|
||||
},
|
||||
{
|
||||
"scenario": "lease_reorder_and_stale_version",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 15.225,
|
||||
"attachment_id": "3bdce3c7-d7d5-49fb-9691-96e626317ae1",
|
||||
"old_job_id": "dd3d35b5-14fe-420b-a650-e740bc43489c",
|
||||
"pipeline_version": 2,
|
||||
"manifest_rows": 8
|
||||
},
|
||||
{
|
||||
"scenario": "bounded_retry_and_explicit_replay",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 15.238,
|
||||
"job_id": "444a1bf1-42ed-4123-9ec7-7f614cd6d90f",
|
||||
"failed_attempt": 5,
|
||||
"replayed_attempt": 6
|
||||
},
|
||||
{
|
||||
"scenario": "permanent_error",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 2.604,
|
||||
"job_id": "d21ee0e0-702d-4459-8fd6-5e5f62c6c11d"
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_staging",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 0.144,
|
||||
"attachment_id": "963d2034-021f-492f-81f3-d31431a2bc0f",
|
||||
"accepted_commit": false
|
||||
},
|
||||
{
|
||||
"scenario": "api_crash_after_upload_commit",
|
||||
"run": 3,
|
||||
"result": "PASS",
|
||||
"seconds": 2.141,
|
||||
"attachment_id": "ea757ed8-98a8-4f3b-993e-556347380e26",
|
||||
"accepted_commit": true
|
||||
}
|
||||
],
|
||||
"tested_image_id": "sha256:0b720b9bab11e4a762e8b180b26c290a43503c4db2db3436c201cb08725f24c0",
|
||||
"runtime_source_sha256": "e6460140119a22471f992cd02c721b55463d0810a6a8e50333e193d674c7c79a",
|
||||
"subsequent_fix": "Restore refreshes retention timestamp; Cleanup shares its source lock. Publisher has a physical socket deadline. Verified by storage.py and the additional final-image fault scenarios in pipeline-final.json; job processing, lease and manifest logic unchanged."
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"scenario": "queued_staging_protected",
|
||||
"result": "PASS",
|
||||
"attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b"
|
||||
},
|
||||
{
|
||||
"scenario": "staging_deleted_after_ready",
|
||||
"result": "PASS",
|
||||
"attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b"
|
||||
},
|
||||
{
|
||||
"scenario": "failed_retention_and_missing_source_replay",
|
||||
"result": "PASS",
|
||||
"age_simulated_hours": 25
|
||||
},
|
||||
{
|
||||
"scenario": "verified_restore_then_replay",
|
||||
"result": "PASS",
|
||||
"job_id": "8d6f4591-e10f-4616-b387-fe7ec5020a84"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<policymap>
|
||||
<policy domain="delegate" rights="none" pattern="*"/>
|
||||
<policy domain="filter" rights="none" pattern="*"/>
|
||||
<policy domain="coder" rights="none" pattern="*"/>
|
||||
<policy domain="coder" rights="read|write" pattern="{PNG,JPEG,WEBP,NULL}"/>
|
||||
<policy domain="resource" name="memory" value="256MiB"/>
|
||||
<policy domain="resource" name="map" value="0"/>
|
||||
<policy domain="resource" name="disk" value="256MiB"/>
|
||||
<policy domain="resource" name="thread" value="1"/>
|
||||
<policy domain="resource" name="time" value="60"/>
|
||||
<policy domain="resource" name="list-length" value="4"/>
|
||||
<policy domain="path" rights="none" pattern="@*"/>
|
||||
</policymap>
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create synthetic local broker credentials, preserving existing settings."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
from urllib.parse import quote
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
env=root/'.env.lab5'
|
||||
values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#'))
|
||||
password=values.get('BROKER_PASSWORD')
|
||||
if not password:
|
||||
password=secrets.token_hex(24)
|
||||
with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n')
|
||||
p=root/'secrets/lab5/broker_url'
|
||||
if p.exists(): p.chmod(0o600)
|
||||
p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n')
|
||||
p.chmod(0o444)
|
||||
print('Local broker credentials ready.')
|
||||
@@ -21,3 +21,5 @@ chmod 444 secrets/lab5/*.pem
|
||||
printf '%s\n' 'Local lab5 settings and synthetic keys ready; existing files preserved.'
|
||||
|
||||
python3 scripts/bootstrap-storage.py
|
||||
|
||||
python3 scripts/bootstrap-broker.py
|
||||
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
./scripts/compose.sh stop api lb
|
||||
./scripts/compose.sh restart postgres redis s3
|
||||
./scripts/compose.sh stop api lb worker dispatcher
|
||||
./scripts/compose.sh restart postgres redis s3 broker
|
||||
./scripts/compose.sh run --rm -T s3-init
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/auth"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/chat"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/user"
|
||||
"github.com/google/uuid"
|
||||
stdhttp "net/http"
|
||||
@@ -56,6 +57,9 @@ func run() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") {
|
||||
return runPipeline(cfg, attachmentConfig, os.Args[1:])
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "storage" {
|
||||
return runStorage(cfg, attachmentConfig, os.Args[2:])
|
||||
}
|
||||
@@ -91,6 +95,12 @@ func run() error {
|
||||
chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache)
|
||||
objectStore := objectstore.New(attachmentConfig)
|
||||
attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc)
|
||||
pipelineConfig, err := config.LoadPipeline()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil)
|
||||
attachmentSvc.Configure(tm, outboxSvc, pipelineConfig)
|
||||
|
||||
httpRouter := router.New(
|
||||
middleware.NewAuthMiddleware(authSvc),
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/adapter/broker"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/config"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error {
|
||||
c, e := config.LoadPipeline()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
pool := postgres.New(cfg.Postgres)
|
||||
defer pool.Close()
|
||||
tm := txmanager.New(pool)
|
||||
repo := &storage.AttachmentsRepo{Pool: pool}
|
||||
rabbit := &broker.Rabbit{URL: c.BrokerURL}
|
||||
ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit)
|
||||
svc := attachment.New(repo, objectstore.New(a), a, nil)
|
||||
svc.Configure(tm, ob, c)
|
||||
ob.OnDead = svc.PublishFailed
|
||||
if args[0] == "jobs" || args[0] == "outbox" {
|
||||
if len(args) < 2 {
|
||||
return errors.New("operator action required")
|
||||
}
|
||||
fs := flag.NewFlagSet(args[1], flag.ContinueOnError)
|
||||
idText := fs.String("id", "", "job ID (attachment ID for reprocess)")
|
||||
version := fs.Int("pipeline-version", 2, "new pipeline version")
|
||||
if e = fs.Parse(args[2:]); e != nil {
|
||||
return e
|
||||
}
|
||||
if args[1] == "list" {
|
||||
jobs, e := repo.ListJobs(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(jobs)
|
||||
}
|
||||
id, e := uuid.Parse(*idText)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
switch args[1] {
|
||||
case "redeliver":
|
||||
if os.Getenv("LAB5_FAULTS") != "1" {
|
||||
return errors.New("redeliver is restricted to the explicit fault profile")
|
||||
}
|
||||
var payload []byte
|
||||
e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
j, e := repo.JobTx(ctx, tx, id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
payload, e = json.Marshal(j.Message)
|
||||
return e
|
||||
})
|
||||
if e == nil {
|
||||
e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload})
|
||||
}
|
||||
case "replay":
|
||||
e = svc.Replay(ctx, id)
|
||||
case "reprocess":
|
||||
e = svc.Reprocess(ctx, id, *version)
|
||||
default:
|
||||
return errors.New("unknown operator action")
|
||||
}
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id})
|
||||
}
|
||||
if c.BrokerURL == "" {
|
||||
return errors.New("BROKER_URL_FILE is required")
|
||||
}
|
||||
if args[0] == "worker" {
|
||||
// Recovery continues while the consumer is disconnected or a delivery is paused.
|
||||
recoveryDone := make(chan struct{})
|
||||
defer func() { stop(); <-recoveryDone }()
|
||||
go func() {
|
||||
defer close(recoveryDone)
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if e := svc.Recover(ctx); e != nil {
|
||||
slog.Warn("job_recovery_failed")
|
||||
}
|
||||
if e := svc.Cleanup(ctx); e != nil {
|
||||
slog.Warn("source_cleanup_failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
for ctx.Err() == nil {
|
||||
if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil {
|
||||
slog.Warn("consumer_reconnecting")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
_, e = ob.ProcessOnce(ctx)
|
||||
if e != nil {
|
||||
slog.Warn("dispatcher_retry")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"gitea.privcloud.tech/mtusi2026/internal/config"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
)
|
||||
|
||||
func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
|
||||
@@ -93,6 +95,12 @@ func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
|
||||
pool := postgres.New(cfg.Postgres)
|
||||
defer pool.Close()
|
||||
svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil)
|
||||
jobs, e := config.LoadPipeline()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
tm := txmanager.New(pool)
|
||||
svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs)
|
||||
switch args[0] {
|
||||
case "reconcile":
|
||||
fs := flag.NewFlagSet("reconcile", flag.ContinueOnError)
|
||||
|
||||
@@ -52,6 +52,7 @@ require (
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
|
||||
@@ -94,6 +94,8 @@ github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/Ge
|
||||
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/fault"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
"github.com/google/uuid"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
const Queue = "attachments.v1"
|
||||
|
||||
var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
|
||||
|
||||
type Rabbit struct{ URL string }
|
||||
|
||||
func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) {
|
||||
dial := amqp.DefaultDial(5 * time.Second)
|
||||
if deadline > 0 {
|
||||
dial = func(network, address string) (net.Conn, error) {
|
||||
c, e := net.DialTimeout(network, address, 5*time.Second)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e = c.SetDeadline(time.Now().Add(deadline)); e != nil {
|
||||
c.Close()
|
||||
return nil, e
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
conn, e := amqp.DialConfig(r.URL, amqp.Config{Heartbeat: 10 * time.Second, Dial: dial})
|
||||
if e != nil {
|
||||
return nil, nil, errors.New("broker connection unavailable")
|
||||
}
|
||||
ch, e := conn.Channel()
|
||||
if e != nil {
|
||||
conn.Close()
|
||||
return nil, nil, errors.New("broker channel unavailable")
|
||||
}
|
||||
_, e = ch.QueueDeclare(Queue+".invalid", true, false, false, false, nil)
|
||||
if e == nil {
|
||||
e = ch.ExchangeDeclare("attachments.invalid", "direct", true, false, false, false, nil)
|
||||
}
|
||||
if e == nil {
|
||||
e = ch.QueueBind(Queue+".invalid", "invalid", "attachments.invalid", false, nil)
|
||||
}
|
||||
if e == nil {
|
||||
_, e = ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{"x-dead-letter-exchange": "attachments.invalid", "x-dead-letter-routing-key": "invalid"})
|
||||
}
|
||||
if e != nil {
|
||||
ch.Close()
|
||||
conn.Close()
|
||||
return nil, nil, errors.New("broker topology unavailable")
|
||||
}
|
||||
return conn, ch, nil
|
||||
}
|
||||
func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) error {
|
||||
conn, ch, e := r.connect(20 * time.Second)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer conn.Close()
|
||||
defer ch.Close()
|
||||
if e = ch.Confirm(false); e != nil {
|
||||
return e
|
||||
}
|
||||
confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1))
|
||||
returns := ch.NotifyReturn(make(chan amqp.Return, 1))
|
||||
var job dto.JobMessage
|
||||
if e = json.Unmarshal(m.Payload, &job); e != nil {
|
||||
return e
|
||||
}
|
||||
fault.Hit(ctx, "before_publish", job.JobID)
|
||||
if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: m.Payload}); e != nil {
|
||||
return errors.New("broker publish failed")
|
||||
}
|
||||
select {
|
||||
case c, ok := <-confirms:
|
||||
if !ok || !c.Ack {
|
||||
return errors.New("broker did not confirm")
|
||||
}
|
||||
select {
|
||||
case <-returns:
|
||||
return errors.New("unroutable job")
|
||||
default:
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
return errors.New("confirm timeout")
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
fault.Hit(ctx, "after_publish", job.JobID)
|
||||
return nil
|
||||
}
|
||||
func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error {
|
||||
conn, ch, e := r.connect(0)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer conn.Close()
|
||||
defer ch.Close()
|
||||
if e = ch.Qos(prefetch, 0, false); e != nil {
|
||||
return e
|
||||
}
|
||||
deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
conn.Close()
|
||||
case <-done:
|
||||
}
|
||||
}()
|
||||
defer close(done)
|
||||
tasks := make(chan amqp.Delivery)
|
||||
finished := make(chan struct{}, concurrency)
|
||||
for range concurrency {
|
||||
go func() {
|
||||
defer func() { finished <- struct{}{} }()
|
||||
for d := range tasks {
|
||||
var m dto.JobMessage
|
||||
dec := json.NewDecoder(bytes.NewReader(d.Body))
|
||||
dec.DisallowUnknownFields()
|
||||
e := dec.Decode(&m)
|
||||
if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) {
|
||||
_ = d.Reject(false)
|
||||
continue
|
||||
}
|
||||
if e = process(ctx, m); e != nil {
|
||||
slog.Warn("job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
_ = d.Nack(false, true)
|
||||
continue
|
||||
}
|
||||
fault.Hit(ctx, "before_ack", m.JobID)
|
||||
_ = d.Ack(false)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for d := range deliveries {
|
||||
select {
|
||||
case tasks <- d:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
close(tasks)
|
||||
for range concurrency {
|
||||
<-finished
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return errors.New("broker consumer disconnected")
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -41,7 +43,20 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename"))
|
||||
return
|
||||
}
|
||||
a, e := h.svc.Upload(w, r, user.UserID, id, values[0])
|
||||
if !attachment.ValidFilename(values[0]) {
|
||||
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename"))
|
||||
return
|
||||
}
|
||||
mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if e != nil || (mt != "image/png" && mt != "image/jpeg") {
|
||||
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG"))
|
||||
return
|
||||
}
|
||||
if r.ContentLength > h.svc.Config.MaxBytes {
|
||||
framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large"))
|
||||
return
|
||||
}
|
||||
a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)})
|
||||
if e != nil {
|
||||
if apperr.CodeFromError(e) == apperr.CodeTooManyRequests {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
@@ -49,7 +64,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
framework.JSONError(r.Context(), w, e)
|
||||
return
|
||||
}
|
||||
framework.JSONResponse(w, http.StatusCreated, a)
|
||||
framework.JSONResponse(w, http.StatusAccepted, a)
|
||||
}
|
||||
func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
|
||||
user, e := framework.ParseToken(r, h.provider)
|
||||
@@ -61,27 +76,32 @@ func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
a, e := h.svc.Get(r.Context(), user.UserID, id)
|
||||
|
||||
if !strings.HasSuffix(r.URL.Path, "/content") {
|
||||
a, e := h.svc.Get(r.Context(), user.UserID, id)
|
||||
if e != nil {
|
||||
framework.JSONError(r.Context(), w, e)
|
||||
return
|
||||
}
|
||||
framework.JSONResponse(w, 200, a.Public)
|
||||
return
|
||||
}
|
||||
v := r.URL.Query().Get("variant")
|
||||
if v == "" {
|
||||
v = "original"
|
||||
}
|
||||
if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" {
|
||||
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant"))
|
||||
return
|
||||
}
|
||||
obj, e := h.svc.Content(r.Context(), user.UserID, id, v)
|
||||
if e != nil {
|
||||
framework.JSONError(r.Context(), w, e)
|
||||
return
|
||||
}
|
||||
if !strings.HasSuffix(r.URL.Path, "/content") {
|
||||
framework.JSONResponse(w, 200, a.Public)
|
||||
return
|
||||
}
|
||||
if v := r.URL.Query().Get("variant"); v != "" && v != "original" {
|
||||
framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant"))
|
||||
return
|
||||
}
|
||||
obj, e := h.svc.Store.Get(r.Context(), a.Key)
|
||||
if e != nil {
|
||||
framework.JSONError(r.Context(), w, attachment.Unavailable())
|
||||
return
|
||||
}
|
||||
defer obj.Body.Close()
|
||||
w.Header().Set("Content-Type", string(a.Public.ContentType))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(a.Public.SizeBytes))
|
||||
w.Header().Set("Content-Type", obj.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
// Once streaming has started a transport failure terminates the response; no JSON
|
||||
// error can be appended to an image. Content-Length exposes incomplete downloads.
|
||||
|
||||
@@ -25,10 +25,15 @@ func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*Attachm
|
||||
}
|
||||
func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) {
|
||||
var found bool
|
||||
e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)", key).Scan(&found)
|
||||
e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found)
|
||||
return found, e
|
||||
}
|
||||
func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) {
|
||||
tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing)
|
||||
return tag.RowsAffected() == 1, e
|
||||
}
|
||||
|
||||
func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error {
|
||||
_, e := c.Exec(ctx, "SELECT 1")
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -8,40 +8,34 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
|
||||
"encoding/json"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
|
||||
)
|
||||
|
||||
type AttachmentRecord struct {
|
||||
Public contracts.Attachment
|
||||
Key string
|
||||
Missing bool
|
||||
}
|
||||
type AttachmentRecord = dto.AttachmentRecord
|
||||
type AttachmentsRepo struct{ Pool *pgxpool.Pool }
|
||||
|
||||
const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing"
|
||||
const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys"
|
||||
|
||||
func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) {
|
||||
a := new(AttachmentRecord)
|
||||
p := &a.Public
|
||||
if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing); err != nil {
|
||||
var variants, keys []byte
|
||||
if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "attachment unavailable")
|
||||
}
|
||||
p.Status = "ready"
|
||||
p.Variants = []contracts.Variant{}
|
||||
if err := json.Unmarshal(variants, &p.Variants); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(keys, &a.VariantKeys); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) {
|
||||
return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id))
|
||||
}
|
||||
func (r *AttachmentsRepo) Save(ctx context.Context, conn *pgxpool.Conn, a *AttachmentRecord) error {
|
||||
p := a.Public
|
||||
_, e := conn.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.Key)
|
||||
if e != nil {
|
||||
return apperr.WrapPostgres(e, "failed to save attachment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Session locks span S3 work and the database commit. A lost connection releases
|
||||
// the lock; the mandatory grace period also exceeds the maximum upload lifetime.
|
||||
|
||||
@@ -58,7 +58,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil {
|
||||
if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = tx.Rollback(ctx); err != nil {
|
||||
@@ -75,7 +75,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := repo.CreateIdempotentTx(ctx, tx, req)
|
||||
first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -91,7 +91,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := repo.CreateIdempotentTx(ctx, tx, req)
|
||||
second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -115,7 +115,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = repo.CreateIdempotentTx(ctx, tx, req); err == nil {
|
||||
if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil {
|
||||
t.Fatal("expected injected failure")
|
||||
}
|
||||
if err = tx.Rollback(ctx); err != nil {
|
||||
@@ -134,7 +134,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil {
|
||||
if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil {
|
||||
t.Fatal("transient failure was cached", err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
@@ -257,7 +257,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = repo.CreateIdempotentTx(ctx, tx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}})
|
||||
_, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing attachment FK failure")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) }
|
||||
func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) {
|
||||
return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id))
|
||||
}
|
||||
func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
|
||||
p := a.Public
|
||||
_, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status)
|
||||
return e
|
||||
}
|
||||
func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
|
||||
variants, e := json.Marshal(a.Public.Variants)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
keys, e := json.Marshal(a.VariantKeys)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing)
|
||||
return e
|
||||
}
|
||||
|
||||
const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at`
|
||||
|
||||
func scanJob(row pgx.Row) (*dto.AttachmentJob, error) {
|
||||
j := new(dto.AttachmentJob)
|
||||
m := &j.Message
|
||||
err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt)
|
||||
m.SchemaVersion = 1
|
||||
m.Type = "attachment.process.v1"
|
||||
return j, err
|
||||
}
|
||||
func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) {
|
||||
return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id))
|
||||
}
|
||||
func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
|
||||
m := j.Message
|
||||
_, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt)
|
||||
return e
|
||||
}
|
||||
func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
|
||||
_, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt)
|
||||
return e
|
||||
}
|
||||
func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) {
|
||||
rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until LIMIT 100`)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []uuid.UUID
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
if e = rows.Scan(&id); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
func (r *AttachmentsRepo) AuditTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, action string) error {
|
||||
_, e := tx.Exec(ctx, `INSERT INTO job_audit(job_id,action) VALUES($1,$2)`, id, action)
|
||||
return e
|
||||
}
|
||||
func (r *AttachmentsRepo) ListJobs(ctx context.Context) ([]dto.AttachmentJob, error) {
|
||||
rows, e := r.Pool.Query(ctx, `SELECT `+jobColumns+` FROM attachment_jobs ORDER BY updated_at DESC LIMIT 100`)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []dto.AttachmentJob{}
|
||||
for rows.Next() {
|
||||
j, e := scanJob(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, *j)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
func (r *AttachmentsRepo) FailedOutbox(ctx context.Context) ([]uuid.UUID, error) {
|
||||
rows, e := r.Pool.Query(ctx, `SELECT DISTINCT o.job_id FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE o.status='dead_letter' AND j.state='queued' AND (o.payload->>'attempt')::integer=j.attempt LIMIT 100`)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []uuid.UUID
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
if e = rows.Scan(&id); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) {
|
||||
rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*dto.AttachmentRecord
|
||||
for rows.Next() {
|
||||
a, e := ScanAttachment(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error {
|
||||
_, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size)
|
||||
return e
|
||||
}
|
||||
|
||||
func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) {
|
||||
var dead bool
|
||||
e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead)
|
||||
return dead, e
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
@@ -26,55 +24,49 @@ func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo {
|
||||
return &MessagesRepo{pool: pool}
|
||||
}
|
||||
|
||||
// CreateIdempotentTx serializes only requests sharing an idempotency scope.
|
||||
// The transaction commits both the message and its replay record, or neither.
|
||||
func (r *MessagesRepo) CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) {
|
||||
scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "failed to lock message key")
|
||||
// LockKeyTx serializes persistence for one idempotency scope.
|
||||
func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error {
|
||||
_, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope)
|
||||
if err != nil {
|
||||
return apperr.WrapPostgres(err, "failed to lock message key")
|
||||
}
|
||||
hash := messageHash(req)
|
||||
var hashVersion int
|
||||
var storedHash []byte
|
||||
var messageID uuid.UUID
|
||||
err := tx.QueryRow(ctx, `SELECT request_hash, message_id, hash_version FROM message_idempotency
|
||||
WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`,
|
||||
req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&storedHash, &messageID, &hashVersion)
|
||||
if err == nil {
|
||||
if hashVersion == 1 && len(req.AttachmentIDs) == 0 {
|
||||
hash = sha256.Sum256([]byte(req.Text))
|
||||
} else if hashVersion != 2 {
|
||||
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
|
||||
}
|
||||
if !bytes.Equal(hash[:], storedHash) {
|
||||
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
|
||||
}
|
||||
return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, messageID))
|
||||
return nil
|
||||
}
|
||||
func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) {
|
||||
v := new(dto.MessageReplay)
|
||||
err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
if err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "failed to read message key")
|
||||
}
|
||||
message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: time.Now().UTC().Truncate(time.Microsecond)}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`,
|
||||
message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt); err != nil {
|
||||
return v, nil
|
||||
}
|
||||
func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) {
|
||||
return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id))
|
||||
}
|
||||
func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error {
|
||||
_, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt)
|
||||
if err != nil {
|
||||
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation {
|
||||
return nil, apperr.Wrap(err, apperr.CodeNotFound, "chat not found")
|
||||
return apperr.Wrap(err, apperr.CodeNotFound, "chat not found")
|
||||
}
|
||||
return nil, apperr.WrapPostgres(err, "failed to create message")
|
||||
return apperr.WrapPostgres(err, "failed to create message")
|
||||
}
|
||||
for position, attachmentID := range req.AttachmentIDs {
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, attachmentID, position); err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "failed to link attachment")
|
||||
for position, id := range message.AttachmentIDs {
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil {
|
||||
return apperr.WrapPostgres(err, "failed to link attachment")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version)
|
||||
VALUES($1,'POST',$2,$3,$4,$5,clock_timestamp()+interval '24 hours',2)
|
||||
ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,
|
||||
message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash[:], message.ID); err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "failed to save message key")
|
||||
return nil
|
||||
}
|
||||
func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error {
|
||||
_, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires)
|
||||
if err != nil {
|
||||
return apperr.WrapPostgres(err, "failed to save message key")
|
||||
}
|
||||
return &message, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) {
|
||||
@@ -120,20 +112,10 @@ func scanMessage(row pgx.Row) (*dto.Message, error) {
|
||||
const messageColumns = `id, chat_id, sender_id, text, created_at,
|
||||
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)`
|
||||
|
||||
func messageHash(req dto.CreateMessage) [32]byte {
|
||||
// Field order and a non-nil list define the canonical v2 representation.
|
||||
canonical := struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []uuid.UUID `json:"attachment_ids"`
|
||||
}{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)}
|
||||
encoded, _ := json.Marshal(canonical) // strings and UUIDs have no encoding failures.
|
||||
return sha256.Sum256(encoded)
|
||||
}
|
||||
|
||||
func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) {
|
||||
// Prevent a concurrent integrity update from invalidating the checked rows
|
||||
// until the message and its links have committed together.
|
||||
rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids)
|
||||
rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids)
|
||||
if err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "failed to read attachments")
|
||||
}
|
||||
@@ -141,7 +123,7 @@ func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, i
|
||||
result := make([]dto.MessageAttachment, 0, len(ids))
|
||||
for rows.Next() {
|
||||
var a dto.MessageAttachment
|
||||
if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing); err != nil {
|
||||
if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil {
|
||||
return nil, apperr.WrapPostgres(err, "failed to read attachment")
|
||||
}
|
||||
result = append(result, a)
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"time"
|
||||
)
|
||||
|
||||
type OutboxRepo struct{ Pool *pgxpool.Pool }
|
||||
|
||||
func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) {
|
||||
_, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt)
|
||||
return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e
|
||||
}
|
||||
func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var m outbox.AcquiredMessage
|
||||
if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil {
|
||||
return
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
err = rows.Err()
|
||||
return
|
||||
}
|
||||
func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) {
|
||||
rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED)
|
||||
UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String())
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return scanOutbox(rows)
|
||||
}
|
||||
func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) {
|
||||
rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until<clock_timestamp() LIMIT 100`)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return scanOutbox(rows)
|
||||
}
|
||||
func (r *OutboxRepo) Update(ctx context.Context, m outbox.AcquiredMessage, status outbox.Status, next time.Time, code string) error {
|
||||
tag, e := r.Pool.Exec(ctx, `UPDATE outbox SET status=$3,next_attempt_at=$4,last_error=NULLIF($5,''),locked_until=NULL,updated_at=clock_timestamp() WHERE id=$1 AND token=$2 AND status='processing'`, m.ID, m.Token, status, next, code)
|
||||
if e == nil && tag.RowsAffected() != 1 {
|
||||
return errors.New("outbox lease lost")
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Pipeline struct {
|
||||
Concurrency, Prefetch, MaxAttempts int
|
||||
Timeout, Lease, Retention time.Duration
|
||||
BrokerURL string
|
||||
}
|
||||
|
||||
func LoadPipeline() (*Pipeline, error) {
|
||||
c := &Pipeline{Concurrency: 2, Prefetch: 2, MaxAttempts: 5, Timeout: 60 * time.Second, Lease: 90 * time.Second, Retention: 24 * time.Hour}
|
||||
for k, p := range map[string]*int{"WORKER_CONCURRENCY": &c.Concurrency, "WORKER_PREFETCH": &c.Prefetch, "JOB_MAX_ATTEMPTS": &c.MaxAttempts} {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
n, e := strconv.Atoi(v)
|
||||
if e != nil || n < 1 || n > 64 {
|
||||
return nil, fmt.Errorf("invalid %s", k)
|
||||
}
|
||||
*p = n
|
||||
}
|
||||
}
|
||||
for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
d, e := time.ParseDuration(v)
|
||||
if e != nil || d <= 0 {
|
||||
return nil, fmt.Errorf("invalid %s", k)
|
||||
}
|
||||
*p = d
|
||||
}
|
||||
}
|
||||
if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency {
|
||||
return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency")
|
||||
}
|
||||
if p := os.Getenv("BROKER_URL_FILE"); p != "" {
|
||||
b, e := os.ReadFile(p)
|
||||
if e != nil {
|
||||
return nil, fmt.Errorf("read broker secret: %w", e)
|
||||
}
|
||||
c.BrokerURL = strings.TrimSpace(string(b))
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -1 +1,48 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
|
||||
"github.com/google/uuid"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AttachmentRecord struct {
|
||||
Public contracts.Attachment
|
||||
Key string
|
||||
StagingKey string
|
||||
PipelineVersion int
|
||||
VariantKeys map[string]string
|
||||
Missing bool
|
||||
}
|
||||
type UploadAttachment struct {
|
||||
UserID, ChatID uuid.UUID
|
||||
Filename, ContentType string
|
||||
Body io.Reader
|
||||
}
|
||||
type AttachmentContent struct {
|
||||
Body io.ReadCloser
|
||||
ContentType string
|
||||
Size int64
|
||||
}
|
||||
type JobMessage struct {
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID uuid.UUID `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
type AttachmentJob struct {
|
||||
Message JobMessage
|
||||
State string
|
||||
Token uuid.UUID
|
||||
LeaseUntil *time.Time
|
||||
NextAttemptAt time.Time
|
||||
AttemptLimit int
|
||||
ErrorCode *string
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
@@ -35,4 +35,11 @@ type MessageAttachment struct {
|
||||
ID uuid.UUID
|
||||
ChatID uuid.UUID
|
||||
Missing bool
|
||||
Status string
|
||||
}
|
||||
|
||||
type MessageReplay struct {
|
||||
Hash []byte
|
||||
HashVersion int
|
||||
MessageID uuid.UUID
|
||||
}
|
||||
|
||||
@@ -22,7 +22,10 @@ const (
|
||||
|
||||
// Defines values for AttachmentStatus.
|
||||
const (
|
||||
Ready AttachmentStatus = "ready"
|
||||
Failed AttachmentStatus = "failed"
|
||||
Processing AttachmentStatus = "processing"
|
||||
Queued AttachmentStatus = "queued"
|
||||
Ready AttachmentStatus = "ready"
|
||||
)
|
||||
|
||||
// Defines values for ErrorErrorCode.
|
||||
@@ -52,14 +55,17 @@ const (
|
||||
|
||||
// Defines values for VariantName.
|
||||
const (
|
||||
Crop VariantName = "crop"
|
||||
Optimized VariantName = "optimized"
|
||||
Thumbnail VariantName = "thumbnail"
|
||||
VariantNameCrop VariantName = "crop"
|
||||
VariantNameOptimized VariantName = "optimized"
|
||||
VariantNameThumbnail VariantName = "thumbnail"
|
||||
)
|
||||
|
||||
// Defines values for DownloadAttachmentParamsVariant.
|
||||
const (
|
||||
Original DownloadAttachmentParamsVariant = "original"
|
||||
DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop"
|
||||
DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized"
|
||||
DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original"
|
||||
DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail"
|
||||
)
|
||||
|
||||
// Attachment defines model for Attachment.
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// Package fault implements explicit, file-controlled barriers for the isolated lab.
|
||||
package fault
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Hit(ctx context.Context, point string, id uuid.UUID) {
|
||||
if os.Getenv("LAB5_FAULTS") != "1" {
|
||||
return
|
||||
}
|
||||
p := filepath.Join("/faults", point+"."+id.String())
|
||||
b, e := os.ReadFile(p)
|
||||
if e != nil {
|
||||
p = filepath.Join("/faults", point+".all")
|
||||
b, e = os.ReadFile(p)
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
_, _ = hit.WriteString(id.String() + "\n")
|
||||
_ = hit.Close()
|
||||
switch strings.TrimSpace(string(b)) {
|
||||
case "crash":
|
||||
_ = os.Remove(p)
|
||||
os.Exit(86)
|
||||
case "pause":
|
||||
for {
|
||||
if _, e = os.Stat(p); os.IsNotExist(e) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Error(point string, id uuid.UUID) error {
|
||||
if os.Getenv("LAB5_FAULTS") != "1" {
|
||||
return nil
|
||||
}
|
||||
b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String()))
|
||||
if e != nil {
|
||||
return nil
|
||||
}
|
||||
v := strings.TrimSpace(string(b))
|
||||
if v == "temporary" || v == "permanent" {
|
||||
return errors.New(v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
func run(ctx context.Context, args ...string) error {
|
||||
cmd := exec.CommandContext(ctx, "magick", args...)
|
||||
cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60")
|
||||
cmd.WaitDelay = time.Second
|
||||
if err := cmd.Run(); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("image conversion failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func Validate(ctx context.Context, path string) error {
|
||||
return run(ctx, "-regard-warnings", path, "null:")
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Name, Path, MIME string
|
||||
Width, Height int
|
||||
}
|
||||
|
||||
func fit(w, h, limit int) (int, int) {
|
||||
if w <= limit && h <= limit {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
return limit, max(1, h*limit/w)
|
||||
}
|
||||
return max(1, w*limit/h), limit
|
||||
}
|
||||
func Process(ctx context.Context, source, dir string, version int) ([]Result, error) {
|
||||
normalized := filepath.Join(dir, "normalized.png")
|
||||
if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
f, e := os.Open(normalized)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
cfg, _, e := image.DecodeConfig(f)
|
||||
f.Close()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
tw, th := fit(cfg.Width, cfg.Height, 256)
|
||||
ow, oh := fit(cfg.Width, cfg.Height, 1280)
|
||||
out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}}
|
||||
for _, v := range out {
|
||||
args := []string{normalized}
|
||||
if v.Name == "crop" {
|
||||
side := min(cfg.Width, cfg.Height)
|
||||
args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage")
|
||||
}
|
||||
args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality")
|
||||
quality := "85"
|
||||
if v.Name == "optimized" && version == 1 {
|
||||
quality = "80"
|
||||
}
|
||||
coder := "jpeg:"
|
||||
if v.Name == "optimized" {
|
||||
coder = "webp:"
|
||||
}
|
||||
args = append(args, quality, coder+v.Path)
|
||||
if e = run(ctx, args...); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -67,3 +67,11 @@ func Missing(err error) bool {
|
||||
var api smithy.APIError
|
||||
return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404")
|
||||
}
|
||||
|
||||
func (s *Store) List(ctx context.Context, token string) (*s3.ListObjectsV2Output, error) {
|
||||
q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)}
|
||||
if token != "" {
|
||||
q.ContinuationToken = &token
|
||||
}
|
||||
return s.Client.ListObjectsV2(ctx, q)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"context"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
type TransactionManager interface {
|
||||
RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error
|
||||
}
|
||||
type Outbox interface {
|
||||
CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error)
|
||||
}
|
||||
type ObjectStore interface {
|
||||
Put(context.Context, string, string, io.Reader, int64) error
|
||||
Get(context.Context, string) (*s3.GetObjectOutput, error)
|
||||
Head(context.Context, string) (*s3.HeadObjectOutput, error)
|
||||
Delete(context.Context, string) error
|
||||
Ping(context.Context) error
|
||||
List(context.Context, string) (*s3.ListObjectsV2Output, error)
|
||||
}
|
||||
type Repository interface {
|
||||
SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error
|
||||
|
||||
Ping(context.Context) error
|
||||
Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error)
|
||||
GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error)
|
||||
InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error
|
||||
UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error
|
||||
JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error)
|
||||
InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error
|
||||
SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error
|
||||
ExpiredJobs(context.Context) ([]uuid.UUID, error)
|
||||
PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error)
|
||||
FailedOutbox(context.Context) ([]uuid.UUID, error)
|
||||
ListJobs(context.Context) ([]dto.AttachmentJob, error)
|
||||
RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error)
|
||||
AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error
|
||||
Lock(context.Context, string) (*pgxpool.Conn, func(), error)
|
||||
Exists(context.Context, *pgxpool.Conn, string) (bool, error)
|
||||
Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error)
|
||||
MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error)
|
||||
CheckConn(context.Context, *pgxpool.Conn) error
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
package attachment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/fault"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrBusy = errors.New("job lease held")
|
||||
|
||||
func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) }
|
||||
func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) }
|
||||
func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error {
|
||||
now := time.Now().UTC()
|
||||
j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now}
|
||||
if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.enqueue(ctx, tx, j)
|
||||
}
|
||||
func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error {
|
||||
b, e := json.Marshal(j.Message)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts})
|
||||
return e
|
||||
}
|
||||
func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) {
|
||||
a, e := s.Get(ctx, user, id)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if a.Public.Status != "ready" {
|
||||
return nil, apperr.New(apperr.CodeConflict, "attachment not ready")
|
||||
}
|
||||
key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes)
|
||||
if variant != "original" {
|
||||
key = a.VariantKeys[variant]
|
||||
if key == "" {
|
||||
return nil, apperr.New(apperr.CodeNotFound, "variant not found")
|
||||
}
|
||||
for _, v := range a.Public.Variants {
|
||||
if string(v.Name) == variant {
|
||||
mime = string(v.ContentType)
|
||||
size = int64(v.SizeBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
obj, e := s.Store.Get(ctx, key)
|
||||
if e != nil {
|
||||
return nil, Unavailable()
|
||||
}
|
||||
return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil
|
||||
}
|
||||
func (s *Service) Process(ctx context.Context, m dto.JobMessage) error {
|
||||
var j *dto.AttachmentJob
|
||||
var a *dto.AttachmentRecord
|
||||
claimed := false
|
||||
err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
var e error
|
||||
j, e = s.Repo.JobTx(ctx, tx, m.JobID)
|
||||
if errors.Is(e, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
// Broker data is a notification. Trusted metadata and attempt live in PostgreSQL.
|
||||
if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt {
|
||||
return nil
|
||||
}
|
||||
a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" {
|
||||
return nil
|
||||
}
|
||||
if j.State == "processing" {
|
||||
return ErrBusy
|
||||
}
|
||||
if time.Now().Before(j.NextAttemptAt) {
|
||||
return ErrBusy
|
||||
}
|
||||
j.State = "processing"
|
||||
j.Token = uuid.New()
|
||||
lease := time.Now().Add(s.Jobs.Lease)
|
||||
j.LeaseUntil = &lease
|
||||
j.UpdatedAt = time.Now().UTC()
|
||||
a.Public.Status = "processing"
|
||||
a.Public.UpdatedAt = j.UpdatedAt
|
||||
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
|
||||
return e
|
||||
}
|
||||
claimed = true
|
||||
return nil
|
||||
})
|
||||
if err != nil || !claimed {
|
||||
return err
|
||||
}
|
||||
s.Fault(ctx, "after_claim", m.JobID)
|
||||
task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout)
|
||||
defer cancel()
|
||||
result, e := s.effects(task, j, a)
|
||||
if e != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return s.finish(ctx, j, nil, e)
|
||||
}
|
||||
s.Fault(ctx, "after_objects", m.JobID)
|
||||
if err = s.finish(ctx, j, result, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
s.Fault(ctx, "after_result_commit", m.JobID)
|
||||
return nil
|
||||
}
|
||||
|
||||
type processed struct {
|
||||
Key string
|
||||
Variants []contracts.Variant
|
||||
Keys map[string]string
|
||||
}
|
||||
type permanentError struct{ code string }
|
||||
|
||||
func (e permanentError) Error() string { return e.code }
|
||||
func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) {
|
||||
dir, e := os.MkdirTemp("", "pipeline-*")
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
key := a.StagingKey
|
||||
if key == "" {
|
||||
key = a.Key
|
||||
}
|
||||
obj, e := s.Store.Get(ctx, key)
|
||||
if e != nil {
|
||||
if objectstore.Missing(e) {
|
||||
return nil, permanentError{"source_missing"}
|
||||
}
|
||||
return nil, e
|
||||
}
|
||||
defer obj.Body.Close()
|
||||
path := filepath.Join(dir, "source")
|
||||
f, e := os.Create(path)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
hash := sha256.New()
|
||||
n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1))
|
||||
f.Close()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 {
|
||||
return nil, permanentError{"source_corrupt"}
|
||||
}
|
||||
f, e = os.Open(path)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
e = s.Validate(ctx, f, string(a.Public.ContentType))
|
||||
f.Close()
|
||||
if e != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, permanentError{"invalid_image"}
|
||||
}
|
||||
s.Fault(ctx, "before_objects", j.Message.JobID)
|
||||
if e := fault.Error("before_objects", j.Message.JobID); e != nil {
|
||||
if e.Error() == "permanent" {
|
||||
return nil, permanentError{"invalid_image"}
|
||||
}
|
||||
return nil, e
|
||||
}
|
||||
converted, e := imageprocessor.Process(ctx, path, dir, j.Message.PipelineVersion)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion)
|
||||
result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}}
|
||||
f, e = os.Open(path)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n)
|
||||
f.Close()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
s.Fault(ctx, "after_original_put", j.Message.JobID)
|
||||
for _, v := range converted {
|
||||
b, e := os.ReadFile(v.Path)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
sum := fmt.Sprintf("%x", sha256.Sum256(b))
|
||||
key := prefix + v.Name + "/" + sum
|
||||
if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
result.Keys[v.Name] = key
|
||||
result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error {
|
||||
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion {
|
||||
return nil
|
||||
}
|
||||
if cause != nil {
|
||||
return s.retry(ctx, tx, j, a, cause)
|
||||
}
|
||||
|
||||
if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil {
|
||||
return e
|
||||
}
|
||||
for _, v := range result.Variants {
|
||||
if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
j.State = "ready"
|
||||
j.ErrorCode = nil
|
||||
j.LeaseUntil = nil
|
||||
j.UpdatedAt = time.Now().UTC()
|
||||
a.Missing = false
|
||||
a.Key = result.Key
|
||||
a.VariantKeys = result.Keys
|
||||
a.Public.Variants = result.Variants
|
||||
a.Public.Status = "ready"
|
||||
a.Public.ErrorCode = nil
|
||||
a.Public.UpdatedAt = j.UpdatedAt
|
||||
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.Repo.UpdateTx(ctx, tx, a)
|
||||
})
|
||||
}
|
||||
func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error {
|
||||
code := "processing_failed"
|
||||
var permanent permanentError
|
||||
terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit
|
||||
if permanent.code != "" {
|
||||
code = permanent.code
|
||||
}
|
||||
if terminal && permanent.code == "" {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
j.LeaseUntil = nil
|
||||
j.Token = uuid.New()
|
||||
j.UpdatedAt = time.Now().UTC()
|
||||
j.ErrorCode = &code
|
||||
if terminal {
|
||||
j.State = "failed"
|
||||
a.Public.Status = "failed"
|
||||
a.Public.ErrorCode = &code
|
||||
} else {
|
||||
j.State = "queued"
|
||||
j.Message.Attempt++
|
||||
j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1))
|
||||
a.Public.Status = "queued"
|
||||
a.Public.ErrorCode = nil
|
||||
if e := s.enqueue(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
a.Public.UpdatedAt = j.UpdatedAt
|
||||
if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.Repo.UpdateTx(ctx, tx, a)
|
||||
}
|
||||
func (s *Service) Recover(ctx context.Context) error {
|
||||
ids, e := s.Repo.ExpiredJobs(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, id := range ids {
|
||||
e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
j, e := s.Repo.JobTx(ctx, tx, id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) {
|
||||
return nil
|
||||
}
|
||||
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if a.PipelineVersion != j.Message.PipelineVersion {
|
||||
return nil
|
||||
}
|
||||
return s.retry(ctx, tx, j, a, errors.New("lease expired"))
|
||||
})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
dead, e := s.Repo.FailedOutbox(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, id := range dead {
|
||||
if e = s.PublishFailed(ctx, id); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error {
|
||||
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
j, e := s.Repo.JobTx(ctx, tx, id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if j.State != "queued" {
|
||||
return nil
|
||||
}
|
||||
dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !dead {
|
||||
return nil
|
||||
}
|
||||
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if a.PipelineVersion != j.Message.PipelineVersion {
|
||||
return nil
|
||||
}
|
||||
code := "publish_failed"
|
||||
j.State = "failed"
|
||||
j.ErrorCode = &code
|
||||
j.UpdatedAt = time.Now().UTC()
|
||||
a.Public.Status = "failed"
|
||||
a.Public.ErrorCode = &code
|
||||
a.Public.UpdatedAt = j.UpdatedAt
|
||||
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.Repo.UpdateTx(ctx, tx, a)
|
||||
})
|
||||
}
|
||||
func (s *Service) Replay(ctx context.Context, id uuid.UUID) error {
|
||||
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
j, e := s.Repo.JobTx(ctx, tx, id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion {
|
||||
return apperr.New(apperr.CodeConflict, "only current failed job can be replayed")
|
||||
}
|
||||
source := a.StagingKey
|
||||
if source == "" {
|
||||
return errors.New("source unavailable; restore original first")
|
||||
}
|
||||
if _, e = s.Store.Head(ctx, source); e != nil {
|
||||
return errors.New("source unavailable")
|
||||
}
|
||||
j.State = "queued"
|
||||
j.Message.Attempt++
|
||||
j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1
|
||||
j.NextAttemptAt = time.Now().UTC()
|
||||
j.UpdatedAt = j.NextAttemptAt
|
||||
j.Token = uuid.New()
|
||||
j.ErrorCode = nil
|
||||
j.LeaseUntil = nil
|
||||
a.Public.Status = "queued"
|
||||
a.Public.ErrorCode = nil
|
||||
a.Public.UpdatedAt = j.UpdatedAt
|
||||
if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.enqueue(ctx, tx, j)
|
||||
})
|
||||
}
|
||||
func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error {
|
||||
if version < 1 || version > 2 {
|
||||
return errors.New("supported pipeline versions: 1, 2")
|
||||
}
|
||||
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
a, e := s.Repo.GetTx(ctx, tx, id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if version <= a.PipelineVersion || a.Public.Status != "ready" {
|
||||
return errors.New("requires ready attachment and newer version")
|
||||
}
|
||||
a.PipelineVersion = version
|
||||
a.StagingKey = a.Key
|
||||
a.Public.Status = "queued"
|
||||
a.Public.Variants = []contracts.Variant{}
|
||||
a.VariantKeys = map[string]string{}
|
||||
a.Public.UpdatedAt = time.Now().UTC()
|
||||
if e = s.Repo.UpdateTx(ctx, tx, a); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.newJob(ctx, tx, a)
|
||||
})
|
||||
}
|
||||
func (s *Service) Cleanup(ctx context.Context) error {
|
||||
records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, record := range records {
|
||||
// Serialize deletion with operator Restore and the initial upload handoff.
|
||||
_, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey)
|
||||
if lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
a, e := s.Repo.GetTx(ctx, tx, record.Public.Id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" {
|
||||
return nil
|
||||
}
|
||||
if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) {
|
||||
return nil
|
||||
}
|
||||
if !strings.HasPrefix(a.StagingKey, "results/") {
|
||||
if e = s.Store.Delete(ctx, a.StagingKey); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
a.StagingKey = ""
|
||||
return s.Repo.UpdateTx(ctx, tx, a)
|
||||
})
|
||||
unlock()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func managedPipelineKey(key string) bool {
|
||||
return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/")
|
||||
}
|
||||
@@ -10,8 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
|
||||
)
|
||||
@@ -38,17 +39,17 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
|
||||
result.Apply = apply
|
||||
result.GraceSeconds = s.Config.Grace.Seconds()
|
||||
defer func() { result.DurationMS = time.Since(start).Milliseconds() }()
|
||||
if err = s.Repo.Pool.Ping(ctx); err != nil {
|
||||
if err = s.Repo.Ping(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
result.S3Requests++
|
||||
if err = s.Store.Ping(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
pager := s3.NewListObjectsV2Paginator(s.Store.Client, &s3.ListObjectsV2Input{Bucket: &s.Store.Bucket, Prefix: aws.String(Prefix), MaxKeys: aws.Int32(100)})
|
||||
for pager.HasMorePages() {
|
||||
token := ""
|
||||
for {
|
||||
result.S3Requests++
|
||||
page, e := pager.NextPage(ctx)
|
||||
page, e := s.Store.List(ctx, token)
|
||||
if e != nil {
|
||||
return result, e
|
||||
}
|
||||
@@ -92,7 +93,7 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
|
||||
return nil
|
||||
}
|
||||
result.DBChecks++
|
||||
if _, e = conn.Exec(ctx, "SELECT 1"); e != nil {
|
||||
if e = s.Repo.CheckConn(ctx, conn); e != nil {
|
||||
return e
|
||||
}
|
||||
result.S3Requests++
|
||||
@@ -106,7 +107,12 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
|
||||
return result, e
|
||||
}
|
||||
}
|
||||
if !aws.ToBool(page.IsTruncated) {
|
||||
break
|
||||
}
|
||||
token = aws.ToString(page.NextContinuationToken)
|
||||
}
|
||||
|
||||
after := uuid.Nil
|
||||
for {
|
||||
result.DBChecks++
|
||||
@@ -120,6 +126,9 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult,
|
||||
for _, a := range page {
|
||||
result.Metadata++
|
||||
after = a.Public.Id
|
||||
if a.Public.Status != "ready" {
|
||||
continue
|
||||
}
|
||||
result.LockAttempts++
|
||||
conn, unlock, e := s.Repo.Lock(ctx, a.Key)
|
||||
if e != nil {
|
||||
@@ -201,5 +210,19 @@ func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error
|
||||
return fmt.Errorf("stored original verification failed")
|
||||
}
|
||||
_, e = s.Repo.MarkMissing(ctx, conn, id, false)
|
||||
return e
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
current, e := s.Repo.GetTx(ctx, tx, id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if current.Public.Status == "failed" {
|
||||
current.StagingKey = current.Key
|
||||
current.Public.UpdatedAt = time.Now().UTC()
|
||||
return s.Repo.UpdateTx(ctx, tx, current)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,8 +8,7 @@ import (
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -18,28 +17,31 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/config"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/gen/contracts"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const Prefix = "originals/"
|
||||
const StagingPrefix = "staging/"
|
||||
|
||||
type Membership interface {
|
||||
GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error)
|
||||
}
|
||||
type Service struct {
|
||||
Repo *storage.AttachmentsRepo
|
||||
Store *objectstore.Store
|
||||
Repo Repository
|
||||
Store ObjectStore
|
||||
Config *config.Attachment
|
||||
chats Membership
|
||||
tm TransactionManager
|
||||
outbox Outbox
|
||||
Jobs *config.Pipeline
|
||||
uploads, decoders chan struct{}
|
||||
}
|
||||
|
||||
func New(repo *storage.AttachmentsRepo, store *objectstore.Store, c *config.Attachment, chats Membership) *Service {
|
||||
func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service {
|
||||
return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)}
|
||||
}
|
||||
func ValidFilename(s string) bool {
|
||||
@@ -53,21 +55,15 @@ func ValidFilename(s string) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid.UUID, filename string) (*contracts.Attachment, error) {
|
||||
ctx := r.Context()
|
||||
if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: chat}); e != nil {
|
||||
func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) {
|
||||
s.tm = tm
|
||||
s.outbox = outbox
|
||||
s.Jobs = jobs
|
||||
}
|
||||
func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (*contracts.Attachment, error) {
|
||||
if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if !ValidFilename(filename) {
|
||||
return nil, apperr.New(apperr.CodeBadRequest, "invalid filename")
|
||||
}
|
||||
mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if e != nil || (mt != "image/png" && mt != "image/jpeg") {
|
||||
return nil, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")
|
||||
}
|
||||
if r.ContentLength > s.Config.MaxBytes {
|
||||
return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
|
||||
}
|
||||
select {
|
||||
case s.uploads <- struct{}{}:
|
||||
defer func() { <-s.uploads }()
|
||||
@@ -80,23 +76,22 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
r.Body = http.MaxBytesReader(w, r.Body, s.Config.MaxBytes)
|
||||
hash := sha256.New()
|
||||
size, e := io.Copy(io.MultiWriter(f, hash), r.Body)
|
||||
size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1))
|
||||
if e != nil {
|
||||
if _, ok := e.(*http.MaxBytesError); ok {
|
||||
return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
|
||||
}
|
||||
return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload")
|
||||
}
|
||||
if e = s.Validate(ctx, f, mt); e != nil {
|
||||
if size > s.Config.MaxBytes {
|
||||
return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large")
|
||||
}
|
||||
if e = s.Validate(ctx, f, req.ContentType); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
id := uuid.New()
|
||||
key := Prefix + id.String()
|
||||
a := &storage.AttachmentRecord{Key: key, Public: contracts.Attachment{Id: id, ChatId: chat, OwnerId: user, Filename: filename, ContentType: contracts.AttachmentContentType(mt), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "ready", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}}
|
||||
conn, unlock, e := s.Repo.Lock(ctx, key)
|
||||
key := StagingPrefix + id.String()
|
||||
a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}}
|
||||
_, unlock, e := s.Repo.Lock(ctx, key)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
@@ -104,14 +99,19 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid
|
||||
if _, e = f.Seek(0, 0); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e = s.Store.Put(ctx, key, mt, f, size); e != nil {
|
||||
if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil {
|
||||
return nil, Unavailable()
|
||||
}
|
||||
// Never blindly compensate a failed/ambiguous COMMIT with DELETE: a committed
|
||||
// row may already reference this key. Reconciliation decides after grace.
|
||||
if e = s.Repo.Save(ctx, conn, a); e != nil {
|
||||
s.Fault(ctx, "after_staging", id)
|
||||
if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
|
||||
if e := s.Repo.InsertTx(ctx, tx, a); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.newJob(ctx, tx, a)
|
||||
}); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
s.Fault(ctx, "after_upload_commit", id)
|
||||
return &a.Public, nil
|
||||
}
|
||||
func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error {
|
||||
@@ -134,12 +134,12 @@ func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error {
|
||||
if _, e = f.Seek(0, 0); e != nil {
|
||||
return e
|
||||
}
|
||||
if _, _, e = image.Decode(f); e != nil {
|
||||
if e = validateImage(ctx, f.Name()); e != nil {
|
||||
return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image")
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.AttachmentRecord, error) {
|
||||
func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) {
|
||||
a, e := s.Repo.Get(ctx, id)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
@@ -154,6 +154,9 @@ func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.Attachm
|
||||
}
|
||||
func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") }
|
||||
func ManagedKey(key string) bool {
|
||||
if managedPipelineKey(key) {
|
||||
return true
|
||||
}
|
||||
if !strings.HasPrefix(key, Prefix) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/dto"
|
||||
"gitea.privcloud.tech/mtusi2026/internal/pkg/apperr"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CreateIdempotentTx performs the message use case inside an existing transaction.
|
||||
func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) {
|
||||
scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey
|
||||
if err := repo.LockKeyTx(ctx, tx, scope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
canonical := struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []uuid.UUID `json:"attachment_ids"`
|
||||
}{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)}
|
||||
encoded, err := json.Marshal(canonical)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash := sha256.Sum256(encoded)
|
||||
replay, err := repo.ReplayTx(ctx, tx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if replay != nil {
|
||||
if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 {
|
||||
hash = sha256.Sum256([]byte(req.Text))
|
||||
} else if replay.HashVersion != 2 {
|
||||
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
|
||||
}
|
||||
if !bytes.Equal(hash[:], replay.Hash) {
|
||||
return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content")
|
||||
}
|
||||
return repo.MessageTx(ctx, tx, replay.MessageID)
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now}
|
||||
if err = repo.InsertMessageTx(ctx, tx, message); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &message, nil
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
@@ -17,7 +18,11 @@ type chatsRepo interface {
|
||||
|
||||
type messagesRepo interface {
|
||||
AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error)
|
||||
CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error)
|
||||
LockKeyTx(context.Context, pgx.Tx, string) error
|
||||
ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error)
|
||||
MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error)
|
||||
InsertMessageTx(context.Context, pgx.Tx, dto.Message) error
|
||||
SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error
|
||||
GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error)
|
||||
ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error)
|
||||
}
|
||||
|
||||
@@ -190,6 +190,9 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M
|
||||
if attachment.ChatID != req.ChatID {
|
||||
return apperr.New(apperr.CodeNotFound, "attachment not found")
|
||||
}
|
||||
if attachment.Status == "failed" {
|
||||
return apperr.New(apperr.CodeConflict, "failed attachment")
|
||||
}
|
||||
if attachment.Missing {
|
||||
return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable")
|
||||
}
|
||||
@@ -197,7 +200,7 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M
|
||||
}
|
||||
|
||||
var err error
|
||||
message, err = c.messagesRepo.CreateIdempotentTx(ctx, tx, req)
|
||||
message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req)
|
||||
return err
|
||||
})
|
||||
return message, err
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type API interface {
|
||||
CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error)
|
||||
Create(context.Context, CreateMessageReq) (CreateMessageRes, error)
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type CreateMessageReq struct {
|
||||
ID uuid.UUID
|
||||
CreatedAt time.Time
|
||||
Transport Transport
|
||||
AggregateType string
|
||||
AggregateID uuid.UUID
|
||||
MessageType string
|
||||
IdempotencyKey *string
|
||||
SchemaVersion int
|
||||
CausationID *uuid.UUID
|
||||
Metadata map[string]string
|
||||
Payload []byte
|
||||
MaxAttempts int
|
||||
Status Status
|
||||
}
|
||||
|
||||
type CreateMessageRes struct {
|
||||
ID uuid.UUID
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type MarkPublishedMessageReq struct {
|
||||
ID uuid.UUID
|
||||
UpdatedAt time.Time
|
||||
PublishedAt time.Time
|
||||
}
|
||||
|
||||
type MarkFailedMessageReq struct {
|
||||
ID uuid.UUID
|
||||
UpdatedAt time.Time
|
||||
NextAttemptAt time.Time
|
||||
LastError string
|
||||
}
|
||||
|
||||
type MarkDeadLetterMessageReq struct {
|
||||
ID uuid.UUID
|
||||
UpdatedAt time.Time
|
||||
DeadLetterAt time.Time
|
||||
LastError string
|
||||
}
|
||||
|
||||
type AcquireMessagesForProcessingReq struct {
|
||||
Limit uint
|
||||
LockDuration time.Duration
|
||||
}
|
||||
|
||||
type AcquiredMessage struct {
|
||||
Token uuid.UUID
|
||||
ID uuid.UUID
|
||||
Transport Transport
|
||||
AggregateType string
|
||||
AggregateID uuid.UUID
|
||||
MessageType string
|
||||
SchemaVersion int
|
||||
CorrelationID uuid.UUID
|
||||
CausationID *uuid.UUID
|
||||
Status Status
|
||||
Metadata map[string]string
|
||||
Payload []byte
|
||||
Attempts int
|
||||
MaxAttempts int
|
||||
LastError *string
|
||||
LockedAt *time.Time
|
||||
LockedUntil *time.Time
|
||||
NextAttemptAt *time.Time
|
||||
PublishedAt *time.Time
|
||||
DeadLetterAt *time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt *time.Time
|
||||
}
|
||||
|
||||
type RecoverStuckMessagesAsFailedReq struct {
|
||||
UpdatedAt time.Time
|
||||
NextAttemptAt time.Time
|
||||
LastError string
|
||||
Limit uint
|
||||
}
|
||||
|
||||
type RecoverStuckMessagesAsDeadLetterReq struct {
|
||||
UpdatedAt time.Time
|
||||
DeadLetterAt time.Time
|
||||
LastError string
|
||||
Limit uint
|
||||
}
|
||||
|
||||
type DeliveryMessage struct {
|
||||
ID uuid.UUID
|
||||
Transport Transport
|
||||
AggregateType string
|
||||
AggregateID uuid.UUID
|
||||
MessageType string
|
||||
SchemaVersion int
|
||||
CorrelationID uuid.UUID
|
||||
CausationID *uuid.UUID
|
||||
Payload []byte
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
type ProcessOnceResult struct {
|
||||
RecoveredAsFailed int64
|
||||
RecoveredAsDeadLetter int64
|
||||
|
||||
Acquired int
|
||||
Published int
|
||||
Failed int
|
||||
DeadLettered int
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package outbox
|
||||
|
||||
const (
|
||||
StatusCreated Status = "created"
|
||||
StatusPending Status = "pending"
|
||||
StatusProcessing Status = "processing"
|
||||
StatusPublished Status = "published"
|
||||
StatusFailed Status = "failed"
|
||||
StatusDeadLetter Status = "dead_letter"
|
||||
)
|
||||
|
||||
type Status string
|
||||
|
||||
func (s Status) String() string {
|
||||
return string(s)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Adapted from mamont-optics/application/internal/service/outbox.
|
||||
// Retains transactional creation, batch acquisition, bounded retries and lease recovery.
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error)
|
||||
AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error)
|
||||
Expired(context.Context) ([]AcquiredMessage, error)
|
||||
Update(context.Context, AcquiredMessage, Status, time.Time, string) error
|
||||
}
|
||||
type TransportPublisher interface {
|
||||
Publish(context.Context, DeliveryMessage) error
|
||||
}
|
||||
type TransactionManager interface {
|
||||
RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error
|
||||
}
|
||||
type Svc struct {
|
||||
repo Repository
|
||||
tm TransactionManager
|
||||
publisher TransportPublisher
|
||||
OnDead func(context.Context, uuid.UUID) error
|
||||
}
|
||||
|
||||
func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc {
|
||||
return &Svc{repo: repo, tm: tm, publisher: publisher}
|
||||
}
|
||||
func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) {
|
||||
if r.ID == uuid.Nil {
|
||||
r.ID = uuid.New()
|
||||
}
|
||||
if r.CreatedAt.IsZero() {
|
||||
r.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
if r.Status == "" {
|
||||
r.Status = StatusPending
|
||||
}
|
||||
if r.MaxAttempts == 0 {
|
||||
r.MaxAttempts = 5
|
||||
}
|
||||
return s.repo.CreateTx(ctx, tx, r)
|
||||
}
|
||||
func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) {
|
||||
err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e })
|
||||
return
|
||||
}
|
||||
func Backoff(attempt int) time.Duration {
|
||||
n := min(max(attempt-1, 0), 5)
|
||||
cap := min(time.Second*time.Duration(1<<n), 30*time.Second)
|
||||
return cap/2 + time.Duration(rand.Int64N(int64(cap/2)+1))
|
||||
}
|
||||
func (s *Svc) ProcessOnce(ctx context.Context) (result ProcessOnceResult, err error) {
|
||||
expired, e := s.repo.Expired(ctx)
|
||||
if e != nil {
|
||||
return result, e
|
||||
}
|
||||
for _, m := range expired {
|
||||
if e = s.fail(ctx, m, "publish_lease_expired"); e != nil {
|
||||
return result, e
|
||||
}
|
||||
}
|
||||
messages, e := s.repo.AcquireManyForProcessing(ctx, AcquireMessagesForProcessingReq{Limit: 1, LockDuration: 30 * time.Second})
|
||||
if e != nil {
|
||||
return result, e
|
||||
}
|
||||
for _, m := range messages {
|
||||
result.Acquired++
|
||||
// Publish payload is the job contract, never the outbox envelope.
|
||||
if s.publisher == nil {
|
||||
return result, errors.New("publisher unavailable")
|
||||
}
|
||||
e = s.publisher.Publish(ctx, DeliveryMessage{ID: m.ID, AggregateID: m.AggregateID, Payload: m.Payload})
|
||||
if e != nil {
|
||||
if e = s.fail(ctx, m, "publish_failed"); e != nil {
|
||||
return result, e
|
||||
}
|
||||
result.Failed++
|
||||
continue
|
||||
}
|
||||
if e = s.repo.Update(ctx, m, StatusPublished, time.Now().UTC(), ""); e != nil {
|
||||
return result, e
|
||||
}
|
||||
result.Published++
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (s *Svc) fail(ctx context.Context, m AcquiredMessage, code string) error {
|
||||
status := StatusFailed
|
||||
if m.Attempts >= m.MaxAttempts {
|
||||
status = StatusDeadLetter
|
||||
}
|
||||
if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil {
|
||||
return err
|
||||
}
|
||||
if status == StatusDeadLetter && s.OnDead != nil {
|
||||
var j struct {
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
}
|
||||
if err := json.Unmarshal(m.Payload, &j); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.OnDead(ctx, j.JobID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package outbox
|
||||
|
||||
type Transport string
|
||||
|
||||
const TransportAMQP Transport = "amqp"
|
||||
|
||||
func (t Transport) String() string { return string(t) }
|
||||
@@ -0,0 +1,41 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')),
|
||||
ADD COLUMN error_code text,
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
|
||||
ADD COLUMN staging_key text NOT NULL DEFAULT '',
|
||||
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
|
||||
ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}';
|
||||
UPDATE attachments SET staging_key=object_key;
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0),
|
||||
attempt_limit integer NOT NULL, state text NOT NULL,
|
||||
token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL,
|
||||
error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL,
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until);
|
||||
CREATE TABLE outbox (
|
||||
id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
payload jsonb NOT NULL, status text NOT NULL,
|
||||
attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL,
|
||||
token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL,
|
||||
last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at);
|
||||
CREATE TABLE job_audit (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Existing lab4 originals become durable sources for the first asynchronous run.
|
||||
INSERT INTO attachment_jobs
|
||||
SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments;
|
||||
INSERT INTO outbox
|
||||
SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at),
|
||||
'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs;
|
||||
|
||||
---- create above / drop below ----
|
||||
DROP TABLE job_audit;
|
||||
DROP TABLE outbox;
|
||||
DROP TABLE attachment_jobs;
|
||||
ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys;
|
||||
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
|
||||
variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')),
|
||||
object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL,
|
||||
PRIMARY KEY(attachment_id,pipeline_version,variant)
|
||||
);
|
||||
---- create above / drop below ----
|
||||
DROP TABLE attachment_variants;
|
||||
@@ -69,11 +69,12 @@ func TestAttachmentContractAndErrors(t *testing.T) {
|
||||
Sha string `json:"sha256"`
|
||||
Size int `json:"size_bytes"`
|
||||
}
|
||||
decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 201, &att)
|
||||
decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att)
|
||||
hash := sha256.Sum256(data)
|
||||
if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) {
|
||||
t.Fatal("original metadata mismatch")
|
||||
}
|
||||
waitAttachmentReady(t, api, alice, att.ID)
|
||||
content := "/api/v1/attachments/" + att.ID + "/content"
|
||||
got := api.request(t, "GET", content, bob, "")
|
||||
if got.status != 200 || !bytes.Equal(got.body, data) {
|
||||
@@ -115,7 +116,7 @@ func TestAttachmentContractAndErrors(t *testing.T) {
|
||||
boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk))
|
||||
boundary = append(boundary, data[len(data)-12:]...)
|
||||
var boundaryAtt map[string]any
|
||||
decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 201, &boundaryAtt)
|
||||
decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt)
|
||||
|
||||
oversized := make([]byte, 10485761)
|
||||
copy(oversized, data)
|
||||
@@ -127,7 +128,7 @@ func TestAttachmentContractAndErrors(t *testing.T) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var jpegAtt map[string]any
|
||||
decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 201, &jpegAtt)
|
||||
decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt)
|
||||
imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}`
|
||||
var message map[string]any
|
||||
key := uuid.NewString()
|
||||
@@ -145,7 +146,9 @@ func TestAttachmentContractAndErrors(t *testing.T) {
|
||||
for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} {
|
||||
assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request")
|
||||
}
|
||||
assertError(t, api.request(t, "GET", content+"?variant=thumbnail", bob, ""), 400, "invalid_request")
|
||||
if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 {
|
||||
t.Fatal("thumbnail unavailable")
|
||||
}
|
||||
}
|
||||
func TestConcurrentAttachmentMessage(t *testing.T) {
|
||||
api := newAPI(t)
|
||||
@@ -154,7 +157,7 @@ func TestConcurrentAttachmentMessage(t *testing.T) {
|
||||
var att struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 201, &att)
|
||||
decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att)
|
||||
key := uuid.NewString()
|
||||
body := `{"text":"","attachment_ids":["` + att.ID + `"]}`
|
||||
type result struct {
|
||||
@@ -202,3 +205,22 @@ func TestConcurrentAttachmentMessage(t *testing.T) {
|
||||
t.Fatal("duplicate history")
|
||||
}
|
||||
}
|
||||
|
||||
func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
var att struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att)
|
||||
if att.Status == "ready" {
|
||||
return
|
||||
}
|
||||
if att.Status == "failed" {
|
||||
t.Fatal("attachment failed")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("attachment readiness deadline")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Проверки решения lab5
|
||||
|
||||
Публичные `tests/` не изменены. Собственные проверки используют их HTTP-клиент и создают только синтетические данные в локальном стенде.
|
||||
|
||||
- `make test-images`: реальные форматы и размеры вариантов, floor/no-upscale, EXIF orientation, белый фон alpha, центральный crop, hashes, ACL и отрицательные входы.
|
||||
- `make test-staging`: защита queued-source, удаление staging после ready, terminal source_missing, 24h retention и восстановление с явным replay. Для retention меняется только возраст одной синтетической записи на 25h; реальный день не ожидается и конфигурация retention не сокращается.
|
||||
- `make test-broker-persistence`: заменяет контейнер брокера при остановленных worker и dispatcher; проверяет, что уже опубликованное сообщение осталось в очереди с тем же volume.
|
||||
- `make test-pipeline`: 13 сценариев × 3 повтора, включая crash, дубли, lease recovery, reorder, устаревший worker и версию pipeline. На время выполнения подключает `compose.faults.yaml`, перезапускает процессы и stores с сохранением volumes. В конце восстанавливает обычный Compose без fault injection.
|
||||
|
||||
Перед запуском нужен `make up`. Проверки, которые останавливают/перезапускают процессы (`test-staging`, `test-pipeline`, persistence/failover), запускаются последовательно. Результаты — `evidence/*.json`; секреты и токены туда не записываются. Директивы fault injection лежат в игнорируемой `.lab5-faults/`.
|
||||
|
||||
`jobs redeliver --id JOB_UUID` — только инструмент экспериментального профиля с `LAB5_FAULTS=1`, публикующий дополнительную копию сохранённой job. Обычный оператор использует `jobs replay` для failed или `jobs reprocess` для новой версии.
|
||||
|
||||
Исследование и ограничения описаны в [REPORT.md](../REPORT.md).
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a confirmed queued message survives broker container replacement, not only restart."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,check
|
||||
|
||||
def compose(*args):
|
||||
r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150)
|
||||
check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip()
|
||||
|
||||
def queued():
|
||||
r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30)
|
||||
if r.returncode:return 0
|
||||
out=r.stdout
|
||||
for line in out.splitlines():
|
||||
fields=line.split()
|
||||
if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1])
|
||||
return 0
|
||||
|
||||
def main():
|
||||
c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap()
|
||||
try:
|
||||
compose('stop','worker');att,data=c.attachment(alice,chat)
|
||||
deadline=time.monotonic()+20
|
||||
while queued()<1 and time.monotonic()<deadline:time.sleep(.2)
|
||||
check(queued()>=1,'message not durably queued before replacement')
|
||||
published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id'])
|
||||
check(published=='t','outbox not marked published before replacement')
|
||||
compose('stop','dispatcher')
|
||||
old=compose('ps','-q','broker')
|
||||
compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker')
|
||||
new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced')
|
||||
deadline=time.monotonic()+30
|
||||
while queued()<1 and time.monotonic()<deadline:time.sleep(.3)
|
||||
check(queued()>=1,'confirmed queue message lost after replacement')
|
||||
compose('start','worker');c.wait_ready(bob,att)
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement')
|
||||
result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True}
|
||||
(ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity')
|
||||
finally:compose('start','worker','dispatcher')
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Image contract checks using synthetic fixtures and real decoding in ImageMagick."""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zlib
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
def magick(args,data=None):
|
||||
r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30)
|
||||
check(r.returncode==0,'fixture decoder failed')
|
||||
return r.stdout
|
||||
|
||||
def png(w,h,rgba):
|
||||
def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b))
|
||||
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'')
|
||||
|
||||
def main():
|
||||
c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[]
|
||||
jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255)))
|
||||
# EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise.
|
||||
tiff=b'II'+struct.pack('<HI',42,8)+struct.pack('<H',1)+struct.pack('<HHI',0x112,3,1)+struct.pack('<H',6)+b'\0\0'+struct.pack('<I',0)
|
||||
exif=b'Exif\0\0'+tiff;oriented=jpeg[:2]+b'\xff\xe1'+struct.pack('>H',len(exif)+2)+exif+jpeg[2:]
|
||||
for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]:
|
||||
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att)
|
||||
original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed')
|
||||
for v in ready['variants']:
|
||||
b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch')
|
||||
actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split()))
|
||||
check(actual==(v['width'],v['height']),'actual dimensions differ')
|
||||
if v['name']=='thumbnail':
|
||||
check(jpeg_size(b)==dimensions,'thumbnail geometry differs')
|
||||
if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white')
|
||||
if v['name']=='crop':
|
||||
check(actual==(128,128),'crop geometry differs')
|
||||
if name=='center_crop':
|
||||
import re
|
||||
color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b))))
|
||||
check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered')
|
||||
check(b'Exif\0\0' not in b,'EXIF retained in derivative')
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve)
|
||||
results.append({'case':name,'result':'PASS','thumbnail':dimensions})
|
||||
for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]:
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime})
|
||||
(ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs')
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens."""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure, check
|
||||
from image_probe import jpeg_size
|
||||
FAULTS=ROOT/'.lab5-faults'
|
||||
COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')]
|
||||
RESULTS=[]
|
||||
SELECTED=set()
|
||||
|
||||
def compose(*args, allow_failure=False):
|
||||
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180)
|
||||
if p.returncode and not allow_failure:
|
||||
# Infrastructure logs may contain settings; report only action and exit status.
|
||||
raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode))
|
||||
return p
|
||||
|
||||
def sql(query):
|
||||
return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip()
|
||||
|
||||
def uid(value):
|
||||
return str(uuid.UUID(value))
|
||||
|
||||
def job(att,version=1):
|
||||
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version))
|
||||
|
||||
def operator(action,id=None,version=None, allow_failure=False):
|
||||
args=['run','--rm','--no-deps','-T','operator','jobs',action]
|
||||
if id: args+=['--id',uid(id)]
|
||||
if version:args+=['--pipeline-version',str(version)]
|
||||
return compose(*args,allow_failure=allow_failure)
|
||||
|
||||
def wait(predicate,timeout=65):
|
||||
deadline=time.monotonic()+timeout
|
||||
while time.monotonic()<deadline:
|
||||
v=predicate()
|
||||
if v:return v
|
||||
time.sleep(.1)
|
||||
raise Failure('experiment condition timed out')
|
||||
|
||||
def directive(point,id,action):
|
||||
p=FAULTS/(point+'.'+id)
|
||||
Path(str(p)+'.hit').unlink(missing_ok=True)
|
||||
p.write_text(action);p.chmod(0o666)
|
||||
return p
|
||||
|
||||
def hit(p,timeout=65):
|
||||
wait(lambda:Path(str(p)+'.hit').exists(),timeout)
|
||||
return Path(str(p)+'.hit').read_text().strip()
|
||||
|
||||
def metadata(c,actor,att):
|
||||
return c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
|
||||
|
||||
def state(c,actor,att,value,timeout=65):
|
||||
return wait(lambda:(a if (a:=metadata(c,actor,att))['status']==value else None),timeout)
|
||||
|
||||
def ready(c,actor,att,content):
|
||||
a=c.wait_ready(actor,att)
|
||||
check(a['error_code'] is None,'ready has error')
|
||||
original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']
|
||||
check(original==content,'original changed')
|
||||
variants={v['name']:v for v in a['variants']}
|
||||
check(set(variants)=={'thumbnail','crop','optimized'},'variant set differs')
|
||||
for name,v in variants.items():
|
||||
r=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
|
||||
check(hashlib.sha256(r['bytes']).hexdigest()==v['sha256'] and len(r['bytes'])==v['size_bytes'],'variant checksum mismatch')
|
||||
if name!='optimized':check(jpeg_size(r['bytes'])==(v['width'],v['height']),'JPEG dimensions differ')
|
||||
else:check(r['bytes'][:4]==b'RIFF' and r['bytes'][8:12]==b'WEBP','optimized is not WebP')
|
||||
rows=int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))
|
||||
check(rows in (4,8),'manifest has duplicate/missing logical variants')
|
||||
return a
|
||||
|
||||
def record(name,run,fn):
|
||||
if SELECTED and name not in SELECTED:return
|
||||
start=time.monotonic();detail=fn() or {}
|
||||
r={'scenario':name,'run':run,'result':'PASS','seconds':round(time.monotonic()-start,3),**detail}
|
||||
RESULTS.append(r);print(json.dumps(r),flush=True)
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);p.add_argument('--base-url',default='http://localhost:18085');p.add_argument('--output',default='evidence/pipeline.json');p.add_argument('--scenario',action='append',default=[]);args=p.parse_args()
|
||||
SELECTED.update(args.scenario)
|
||||
FAULTS.mkdir(exist_ok=True);FAULTS.chmod(0o777)
|
||||
for f in FAULTS.iterdir():f.unlink()
|
||||
compose('up','-d','--build','--wait','--wait-timeout','180')
|
||||
c=Client(args.base_url,timeout=15);alice,bob,eve,chat=c.bootstrap()
|
||||
def upload_stopped():
|
||||
compose('stop','worker');att,content=c.attachment(alice,chat);return att,content,job(att)
|
||||
try:
|
||||
for run in range(1,args.repeats+1):
|
||||
def recovery():
|
||||
att,data,jid=upload_stopped();c.message(alice,chat,'',attachments=[att['id']]);c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
|
||||
before=sql('SELECT pg_postmaster_start_time()')
|
||||
compose('restart','api','postgres','redis','s3','broker')
|
||||
wait(lambda:health(c),90);compose('run','--rm','--no-deps','-T','s3-init');compose('start','worker');ready(c,bob,att,data)
|
||||
check(sql('SELECT pg_postmaster_start_time()')!=before,'PostgreSQL was not restarted')
|
||||
return {'attachment_id':att['id'],'job_id':jid,'postgres_restarted':True}
|
||||
record('worker_down_and_store_restart',run,recovery)
|
||||
for point in ('before_objects','after_original_put','after_objects','after_result_commit','before_ack'):
|
||||
def crash(point=point):
|
||||
att,data,jid=upload_stopped();marker=directive(point,jid,'crash');compose('start','worker');hit(marker);ready(c,bob,att,data)
|
||||
return {'job_id':jid,'manifest_rows':int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))}
|
||||
record('crash_'+point,run,crash)
|
||||
def duplicates():
|
||||
att,data=c.attachment(alice,chat);first=ready(c,bob,att,data);jid=job(att)
|
||||
for _ in range(3):operator('redeliver',jid)
|
||||
time.sleep(.5);check(metadata(c,bob,att)==first,'ready changed after duplicate')
|
||||
return {'job_id':jid,'copies':3}
|
||||
record('ready_redelivery',run,duplicates)
|
||||
def publish_crash():
|
||||
att,data,jid=upload_stopped();compose('stop','dispatcher');marker=directive('after_publish',jid,'crash')
|
||||
# Existing publish may have completed before stop. Add a second durable notification.
|
||||
sql("UPDATE outbox SET status='pending',next_attempt_at=clock_timestamp() WHERE job_id='%s'"%uid(jid))
|
||||
compose('start','dispatcher');hit(marker);compose('start','worker');ready(c,bob,att,data)
|
||||
wait(lambda:sql("SELECT count(*) FROM outbox WHERE job_id='%s' AND status='published'"%uid(jid))!='0',45)
|
||||
return {'job_id':jid}
|
||||
record('crash_confirm_before_outbox_mark',run,publish_crash)
|
||||
def version_fence():
|
||||
att,data,jid=upload_stopped();marker=directive('after_objects',jid,'pause');compose('start','worker');hit(marker)
|
||||
# Recovery expires the real 8s lease; a newer attempt finishes on the second worker slot.
|
||||
first=ready(c,bob,att,data);check(int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid)))>=2,'lease not recovered')
|
||||
operator('reprocess',att['id'],2);second=ready(c,bob,att,data)
|
||||
check(first['variants']!=second['variants'],'pipeline versions are indistinguishable')
|
||||
operator('redeliver',jid);marker.unlink();time.sleep(1)
|
||||
check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2')
|
||||
check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back')
|
||||
ready(c,bob,att,data)
|
||||
return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8}
|
||||
record('lease_reorder_and_stale_version',run,version_fence)
|
||||
def retries():
|
||||
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65)
|
||||
check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed')
|
||||
return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6}
|
||||
record('bounded_retry_and_explicit_replay',run,retries)
|
||||
def permanent():
|
||||
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid}
|
||||
record('permanent_error',run,permanent)
|
||||
for point in ('after_staging','after_upload_commit'):
|
||||
def api_crash(point=point):
|
||||
if point=='after_upload_commit':compose('stop','dispatcher')
|
||||
marker=directive(point,'all','crash')
|
||||
try:c.attachment(alice,chat)
|
||||
except Failure:pass
|
||||
aid=hit(marker);att={'id':aid}
|
||||
if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists')
|
||||
else:
|
||||
check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached')
|
||||
compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes())
|
||||
wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'}
|
||||
record('api_crash_'+point,run,api_crash)
|
||||
payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS}
|
||||
check(bool(RESULTS),'no scenarios selected')
|
||||
payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab5','--format','{{.Id}}'],text=True).strip()
|
||||
(ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n')
|
||||
finally:
|
||||
for f in FAULTS.iterdir():f.unlink()
|
||||
compose('stop','worker','dispatcher',allow_failure=True)
|
||||
# Restore standard configuration, disabling fault injection and short experiment leases.
|
||||
subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180)
|
||||
|
||||
def health(c):
|
||||
try:c.call('GET','/health/ready',schema='Health');return True
|
||||
except Failure:return False
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Staging protection, retirement, retention and explicit restoration on synthetic data."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,check
|
||||
BASE=[str(ROOT/'scripts/compose.sh')]
|
||||
import subprocess
|
||||
|
||||
def command(*args,fail=False):
|
||||
r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120)
|
||||
if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0])
|
||||
return r
|
||||
|
||||
def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip()
|
||||
def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail)
|
||||
def wait(fn):
|
||||
deadline=time.monotonic()+20
|
||||
while time.monotonic()<deadline:
|
||||
if fn():return
|
||||
time.sleep(.1)
|
||||
raise AssertionError('storage condition timed out')
|
||||
def exists(key):return json.loads(op('storage','object-check','--key',key).stdout)['exists']
|
||||
|
||||
def main():
|
||||
c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[]
|
||||
try:
|
||||
command('stop','worker')
|
||||
att,data=c.attachment(alice,chat);key='staging/'+att['id']
|
||||
check(exists(key),'accepted source missing')
|
||||
result=json.loads(op('storage','reconcile','--apply').stdout)
|
||||
check(result['referenced_objects']>0 and exists(key),'reconciliation removed active source')
|
||||
results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']})
|
||||
command('start','worker');c.wait_ready(bob,att)
|
||||
wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='')
|
||||
check(not exists(key),'ready source staging not retired')
|
||||
results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']})
|
||||
command('stop','worker')
|
||||
failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id'])
|
||||
op('storage','remove-object','--id',failed['id']);command('start','worker')
|
||||
def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed'
|
||||
wait(is_failed)
|
||||
check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent')
|
||||
check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source')
|
||||
op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png')
|
||||
check(exists(key),'restored source missing')
|
||||
# Advance only the synthetic record's age; do not wait a real day or change production retention.
|
||||
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id'])
|
||||
wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='')
|
||||
check(not exists(key),'expired failed source not retired')
|
||||
check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source')
|
||||
results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25})
|
||||
op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed)
|
||||
check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed')
|
||||
results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid})
|
||||
(ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n')
|
||||
print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified')
|
||||
finally:command('start','worker')
|
||||
if __name__=='__main__':main()
|
||||
Reference in New Issue
Block a user