diff --git a/.env.example b/.env.example index eb8a4a0f..0e85d04b 100644 --- a/.env.example +++ b/.env.example @@ -97,10 +97,63 @@ S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID=lab4app S3_SECRET_ACCESS_KEY=lab4app-secret -# Attachment limits from contract 1.4.0. +# Attachment limits from contract 1.5.0. MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 # Presigned upload lifetime and the sweeper's safety window. S3_PRESIGN_TTL=5m S3_SWEEP_INTERVAL=1m S3_SWEEP_GRACE=10m + +# --- Lab5: durable queues and async image processing ------------------------- +# Synthetic broker credentials for the local Compose stand. The relay/worker +# connect inside the compose network; the management UI is published only on +# loopback by compose.yaml. +BROKER_USER=messenger +BROKER_PASSWORD=messenger +BROKER_VHOST=messenger +BROKER_URL=amqp://messenger:messenger@broker:5672/messenger +BROKER_EXCHANGE=attachments.jobs +BROKER_QUEUE=attachment.process.v1 +BROKER_DLX=attachments.dlx +BROKER_DLQ=attachment.process.dead +BROKER_MANAGEMENT_PORT=15672 +BROKER_CPU_LIMIT=0.50 +BROKER_MEM_LIMIT=384M + +# Worker process bounds. Concurrency=1 is the safe default for a 40 MP image on +# the course VM; raise it together with WORKER_MEM_LIMIT. +WORKER_CONCURRENCY=1 +WORKER_PREFETCH=1 +WORKER_CPU_LIMIT=1.00 +WORKER_MEM_LIMIT=1G +WORKER_SHUTDOWN_TIMEOUT=30s +WORKER_HEALTH_PORT=8081 + +# Shared job contract. The API chooses the pipeline version for new uploads. +PROCESSING_PIPELINE_VERSION=1 +PROCESSING_MAX_ATTEMPTS=5 +PROCESSING_JOB_TIMEOUT=120s +PROCESSING_LEASE_TTL=180s +PROCESSING_HEARTBEAT=30s +PROCESSING_RETRY_BASE=2s +PROCESSING_RETRY_CAP=60s + +# Outbox relay and lease reaper. +OUTBOX_POLL_INTERVAL=1s +OUTBOX_CLAIM_TTL=30s +OUTBOX_BATCH=20 +OUTBOX_REAPER_INTERVAL=15s +OUTBOX_RECONCILE_AFTER=5m +OUTBOX_RETENTION=168h +RELAY_CPU_LIMIT=0.25 +RELAY_MEM_LIMIT=128M +RELAY_HEALTH_PORT=8082 + +# Staging cleanup and admission control. +STAGING_ORPHAN_GRACE=1h +STAGING_FAILED_RETENTION=168h +ADMISSION_MAX_PENDING_PER_USER=20 +ADMISSION_MAX_PENDING=200 +ADMISSION_MAX_STAGED_BYTES=2147483648 +ADMISSION_MAX_CONCURRENT_UPLOADS=8 diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index 89c3e684..deff1c1d 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -47,7 +47,7 @@ jobs: tags: mtusi-messenger:ci stand: - name: Lab4 stand scenario + name: Lab5 stand scenario needs: build runs-on: ubuntu-latest timeout-minutes: 20 @@ -68,8 +68,14 @@ jobs: run: make test - name: Attachment negative scenarios run: make test-attachments - - name: Presigned direct upload + - name: Presigned direct upload (async complete) run: make test-presign + - name: Worker process is separate and healthy + run: docker compose --env-file .env ps worker relay broker + - name: Async worker and broker durability + run: make test-queue + - name: Track 2 duplicate and stale-version harness + run: python3 scripts/lab5_track2.py --reps 1 - name: Parallel idempotency through the LB run: python3 student-tests/idempotency_parallel.py --base-url http://localhost:8080 - name: Upload, stop the serving replica, download through another diff --git a/.gitignore b/.gitignore index 6c6271ba..2c1bc700 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ target/ bin/ /messenger /s3init +/worker +/relay +/jobctl obj/ coverage/ coverage.out diff --git a/Dockerfile b/Dockerfile index b2de9f1c..e1f5ae40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,17 +31,29 @@ ENV GOMEMLIMIT=1GiB COPY . . RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -trimpath -buildvcs=false -tags timetzdata \ + go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \ -ldflags="-s -w -X main.version=${VERSION}" \ -o /out/messenger ./cmd/messenger \ && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -trimpath -buildvcs=false -tags timetzdata \ + go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \ -ldflags="-s -w -X main.version=${VERSION}" \ -o /out/migrate ./cmd/migrate \ && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -trimpath -buildvcs=false -tags timetzdata \ + go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \ -ldflags="-s -w -X main.version=${VERSION}" \ - -o /out/s3init ./cmd/s3init + -o /out/s3init ./cmd/s3init \ + && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \ + -ldflags="-s -w -X main.version=${VERSION}" \ + -o /out/worker ./cmd/worker \ + && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \ + -ldflags="-s -w -X main.version=${VERSION}" \ + -o /out/relay ./cmd/relay \ + && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \ + -ldflags="-s -w -X main.version=${VERSION}" \ + -o /out/jobctl ./cmd/jobctl FROM alpine:${ALPINE_VERSION} AS runtime RUN apk add --no-cache ca-certificates wget \ @@ -51,6 +63,9 @@ RUN apk add --no-cache ca-certificates wget \ COPY --from=build --chown=10001:10001 /out/messenger /usr/local/bin/messenger COPY --from=build --chown=10001:10001 /out/migrate /usr/local/bin/migrate COPY --from=build --chown=10001:10001 /out/s3init /usr/local/bin/s3init +COPY --from=build --chown=10001:10001 /out/worker /usr/local/bin/worker +COPY --from=build --chown=10001:10001 /out/relay /usr/local/bin/relay +COPY --from=build --chown=10001:10001 /out/jobctl /usr/local/bin/jobctl COPY --chown=10001:10001 config/dev/config.yaml /etc/messenger/dev/config.yaml USER 10001:10001 diff --git a/Makefile b/Makefile index 4a3ff7c1..64374cb1 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ PYTHON ?= python3 BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= +PIPELINE ?= 2 TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) PYTHON_LOCAL = env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 $(PYTHON) @@ -27,6 +28,15 @@ test-attachments: test-presign: $(PYTHON_LOCAL) student-tests/presign_flow.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-queue: + $(PYTHON_LOCAL) student-tests/lab5_async.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-faults: + $(PYTHON_LOCAL) student-tests/lab5_faults.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-track2: + $(PYTHON_LOCAL) scripts/lab5_track2.py --base-url "$(BASE_URL)" $(TLS_ARGS) + load: $(PYTHON_LOCAL) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -71,6 +81,18 @@ ps: logs: $(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) logs -f --tail 100 +# Operator views: the failed registry is PostgreSQL, the DLQ is a backstop. +jobs-failed: + $(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) run --rm --entrypoint /usr/local/bin/jobctl worker failed + +replay: + @test -n "$(JOB)" || (echo "Set JOB="; exit 2) + $(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) run --rm --entrypoint /usr/local/bin/jobctl worker replay -job "$(JOB)" + +reprocess: + @test -n "$(ATTACHMENT)" || (echo "Set ATTACHMENT="; exit 2) + $(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) run --rm --entrypoint /usr/local/bin/jobctl worker reprocess -attachment "$(ATTACHMENT)" -pipeline "$(PIPELINE)" + .DEFAULT_GOAL := help GO ?= go APP_NAME ?= messenger diff --git a/REPORT.md b/REPORT.md index e98f75e9..539254f9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,218 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кунецкий Владислав, БВТ2303 +- Целевая оценка; выбранный трек 5: **5, трек 2 (доказанная идемпотентность pipeline)** +- Commit/PR: ветка `lab5`; история коммитов содержит перенос lab1–lab4 и базу контракта 1.5.0 +- Использованные источники, библиотеки и помощь ИИ: Go 1.27, PostgreSQL 18, RabbitMQ 4.3.7, SeaweedFS 3.80, HAProxy 3.2, AWS SDK for Go v2, ogen 1.24, sqlc, goose, mockery, `github.com/gen2brain/webp` (libwebp→WASM, CGo-free), `golang.org/x/image`, `github.com/rabbitmq/amqp091-go`. Внешние источники: контракт 1.5.0, `contracts/attachment-states.md`, `contracts/jobs.schema.json`, RabbitMQ quorum queue/delayed retry docs, PostgreSQL `FOR UPDATE SKIP LOCKED`, AWS presigned URL docs. ИИ-ассистент использовался для генерации кода и текста; все сценарии воспроизведены локально, результаты — в `evidence/lab5/`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Engine/Desktop с Compose v2, Python 3.10+ для публичных и студенческих тестов, +Go 1.27 — только для unit/integration тестов и `make generate`. Образ собирает приложение сам. + +```sh +git clone mtusi2026lab5 && cd mtusi2026lab5 +git switch lab5 +cp -n .env.example .env +make check +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# дождаться /health/ready (LB, localhost:8080), затем: +make test +make test-attachments +make test-presign +make test-queue +make test-faults +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Остановка без потери данных: `docker compose --env-file .env down` (named volumes сохраняются). +Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. + +Стенд требует ~3–4 ГБ RAM Docker VM для worker (1G) + broker (384M) + существующих сервисов; +в текущем окружении 2 ГБ, поэтому worker работает с `WORKER_CONCURRENCY=1` и `WORKER_MEM_LIMIT=1G` +как верхний предел, а не резервированием. Для серии с 40 Мп изображением нужен запас VM и замер RSS. + +`make check` проверяет контракт, Python-синтаксис и S3 identities. `make validate` требует +`requirements-dev.txt`: +```sh +python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt +.venv/bin/python scripts/validate_openapi.py +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```mermaid +flowchart LR + C[client] -->|POST bytes| API[api x N] + API -->|validate + spool| API + API -->|Put staging| S3[(SeaweedFS S3)] + API -->|TX: attachment queued + job + outbox| PG[(PostgreSQL)] + RELAY[relay] -->|claim outbox + publisher confirm| MQ[(RabbitMQ quorum + DLQ)] + MQ -->|at-least-once delivery| W[worker] + W -->|read source, verify sha| S3 + W -->|Put original + variants| S3 + W -->|TX: variants + ready/retry/failed| PG + W -->|ack after commit| MQ + API -->|GET status/variants| PG + API -->|GET content| S3 + RELAY -->|lease reaper| PG +``` + +- **API** принимает байты, валидирует PNG/JPEG по содержимому (bytes, пиксели, MIME, filename), + пишет оригинал в приватный S3 staging, затем одной транзакцией создаёт `attachments(status=queued)`, + `attachment_jobs` и `outbox_messages`. Только после этого возвращается **202** со снимком `queued`. + API не публикует в broker и не делает resize. +- **Relay** — отдельный процесс: читает outbox через `FOR UPDATE SKIP LOCKED`, публикует persistent + сообщение с publisher confirm, помечает `published_at` только после ack. Там же lease reaper: + просроченные `processing` возвращаются в `queued` с новым attempt, а исчерпавшие бюджет — в `failed`. + Дополнительно reaper переиздаёт `queued` jobs без unpublished outbox-строки (потерянная доставка, + dead-letter или data loss broker-а); worker CAS делает дубль безопасным. +- **Worker** — отдельный процесс/контейнер. Claim через CAS, heartbeat, обработка, запись S3, + затем одна транзакция результата; ack только после commit. Падение до ack приводит к redelivery; + повтор видит готовый эффект и делает no-op ack. +- **PostgreSQL** — истина о состоянии/намерении. **S3** — байты. **RabbitMQ** — durable transport + at-least-once, не источник истины. Границы: 202 = staging + TX intent; publisher confirm ≠ consumer ack. +- Staging: `staging/uploads//`; presign проверенный объект копируется в + `staging/verified/...`; финальный оригинал — `attachments///original`; + варианты — `attachments///variants/v/.`. +- Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`: PK в `attachment_variants`, + deterministic S3 keys, CAS по lease/attempt. `ready` терминален; старая версия не перезаписывает новую. +- `attachments.pipeline_version` — **опубликованная** версия. `jobctl reprocess` создаёт job новой + версии, но переключает `pipeline_version` и variants только в транзакции успешного завершения; + до этого API продолжает отдавать старую готовую версию. +- Presign — факультативное расширение вне OAS: presign создаёт internal `pending`-строку; complete + проверяет байты, копирует их в server-owned verified staging и возвращает 202 queued. +- Sweeper status-aware: staging queued/processing не удаляется; ready staging удаляется после + безопасной фиксации; failed — после retention; orphan staging — после orphan grace. Referenced set + включает финальные объекты, variants и активный staging; keyset-пагинация, fail-closed. +- Admission control ограничивает число незавершённых rows (включая `pending` presign) на + пользователя/стенд и объём staged bytes (429 `rate_limited`). Проверка повторяется внутри + транзакции под PostgreSQL advisory lock, поэтому параллельные запросы не обходят cap; presign + использует тот же механизм. Число одновременных upload-spool на процесс ограничено + `ADMISSION_MAX_CONCURRENT_UPLOADS`, что ограничивает и временный диск API. + +### Соответствие `.env.example` реальной конфигурации + +`BROKER_URL`, `WORKER_CONCURRENCY`, `WORKER_PREFETCH`, `PROCESSING_*`, `OUTBOX_*`, `STAGING_*`, +`ADMISSION_*` читаются viper-ом (`section.key` → `SECTION_KEY`). `S3_*`, `MAX_UPLOAD_BYTES`, +`MAX_IMAGE_PIXELS`, `POSTGRES_*`, `REDIS_*` сохранены с lab4. Compose подменяет `localhost` +на имена сервисов для контейнеров; значения `.env` остаются локальными для запуска с хоста. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke, upload 201→202, thumbnail | `make test` | PASS lab5: 202→queued→ready, 256×160 JPEG, sha/size | `evidence/lab5/smoke.txt` | +| 202 после durable staging, worker отдельно | `make test-queue` (`lab5_async.py`) | worker stopped: 202 queued, content 409, ссылка на сообщение; после старта ready | `evidence/lab5/async.txt` | +| Broker restart с volume | `lab5_async.py` | upload при остановленном broker → 202; после старта broker+relay ready | `evidence/lab5/async.txt` | +| Полный restart стека | `lab5_async.py` | ready-вложение и original сохранились | `evidence/lab5/async.txt` | +| Варианты: thumbnail/crop/optimized | `make test-queue`/`lab5_variants.py` | thumbnail 256×160 JPEG, crop 128×128 JPEG, optimized WebP 320×200; metadata == bytes; EXIF отсутствует | `evidence/lab5/variants.txt` | +| Состояния queued/processing/ready/failed | `lab5_async.py`, `lab5_faults.py` | queued/ready и failed/error_code наблюдаются через API | `evidence/lab5/async.txt`, `faults.txt` | +| 415 на невалидное изображение | `make test-attachments` | пустое тело, IDAT-CRC, MIME spoof, не-zlib IDAT, пустой растр, недопустимый scanline-фильтр, несовместимые color type/bit depth, дубликаты IHDR, мусор после zlib, битый tRNS, PLTE сверх bit depth, tRNS сверх PLTE — 415 до 202 | `evidence/lab5/attachments-negative.txt` | +| Original неизменен | smoke + variants | скачанные байты и sha совпадают с загруженными | `evidence/lab5/smoke.txt`, `variants.txt` | +| Retry/backoff/classification | `make test-faults` | permanent → failed сразу; transient → 5 attempts, 4 retry events, `retry_exhausted`; DLQ | `evidence/lab5/faults.txt` | +| DLQ/registry + replay | `lab5_faults.py` | failed виден в `jobctl failed`; replay → ready; replay ready refused | `evidence/lab5/faults.txt` | +| Crash после записи объекта до ack | `lab5_faults.py` | redelivery, attempts=2, тот же набор из трёх variants | `evidence/lab5/faults.txt` | +| Crash после claim до side effect | `lab5_faults.py` | lease/redelivery, attempts=2, ready | `evidence/lab5/faults.txt` | +| Crash после commit до ack | `lab5_faults.py` | ready сохранился, повтор — no-op, attempt не вырос | `evidence/lab5/faults.txt` | +| Outbox gap COMMIT→publish | `lab5_faults.py` | unpublished row → relay crash → recovery → published/ready | `evidence/lab5/faults.txt` | +| Lease зависшего processing | `lab5_faults.py` | pause worker → lease expiry → requeue → fence stale → ready | `evidence/lab5/faults.txt` | +| Persistence lab2+ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS с broker/worker/relay в hook | `evidence/lab5/persistence.txt` | +| Failover lab3+ | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 99 успешных записей, 0 ошибок, recovery 0.22 s | `evidence/lab5/failover.txt` | +| Idempotency lab3+ | `make test-idempotency` | 1 accepted, 11 conflicts, stable replay | `evidence/lab5/idempotency.txt` | +| LB исключает not-ready реплику | `make test-lb-readiness` | candidate 200→503, LB DOWN в пределах 30 s, 40 запросов без candidate | `evidence/lab5/lb-readiness-detail.txt` | +| Трек 2: duplicate/stale/version | `python3 scripts/lab5_track2.py --reps 3` | 6/6 PASS: duplicate before effect, v1→v2, stale v1 no-op | `evidence/lab5/track2/` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Параметры кодирования вариантов + +| pipeline_version | thumbnail/crop JPEG | optimized WebP | WebP method | EXIF/alpha | +| --- | --- | --- | --- | --- | +| 1 | quality 85 | quality 80 | 2 | EXIF orientation применяется; metadata удаляется повторным encode; alpha компонуется на белый для JPEG | +| 2 | quality 70 | quality 65 | 4 | то же | + +Параметры закомпилированы в `internal/imageproc.Pipeline` и неизменны для версии; retry не может +получить другие байты. Optimized не увеличивает изображение, fit ≤1280×1280. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Воздействие | Что ожидали | Что наблюдали | +| --- | --- | --- | +| Worker остановлен до upload | 202 queued, content 409, работа не теряется | 202, статус queued >2 s, после старта ready | +| Broker остановлен до upload | 202, outbox intent сохраняется | 202; после старта broker+relay ready | +| Relay убит после COMMIT до publish | outbox остаётся, повтор публикует | unpublished→published, ready, дублей эффекта нет | +| Worker убит после первого variant | redelivery дописывает тот же набор | attempts=2, три variants, original не изменён | +| Worker убит после commit до ack | ready, повтор — no-op | ready, attempt=1 | +| Worker paused (lease) | reaper вернёт работу, stale worker не запишет | requeue, новый attempt, ready | +| Permanent error | failed сразу, error_code, DLQ | failed/injected_failure, attempts=1 | +| Transient errors | 5 attempts, backoff+jitter, failed | attempts=5, `retry_exhausted`, DLQ | +| Failed replay | failed→queued→ready без дубля | ready, один thumbnail | +| v1 ready → reprocess v2 → old v1 delivery | v2 не перезаписывается | pv=2, variants v2, stale v1 ack | +| Reprocess queued при остановленном worker | v2 берёт проверенный staging, а не несуществующий original | pv=2 ready, v1 superseded, событие superseded записано | +| Reprocess v2 падает до публикации | вложение терминально failed, не вечный queued | failed/error_code, active jobs 0 | +| Replay упавшей v2 до публикации | API не показывает stale failed, работа возвращается | job queued, attachment queued, затем ready:v2 | +| Replay старой failed-v1 при наличии v2 | команда отклонена, источник v2 не потерян | v1 failed, v2 queued → ready:v2 | +| Version-stale queued job после публикации v2 | задание терминально superseded, не переиздаётся | claim → superseded, reconcile не создаёт работу | +| Reprocess на неподдерживаемую версию | команда отклонена, v1-задание не тронуто | jobctl exit != 0, v1 queued → ready | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Измерения: recovery времени и counts — из `evidence/lab5/faults.txt`, `track2/runs.csv`. +Стенд: 2 vCPU, 2 ГБ Docker VM; синтетические пользователи/чат; fixture 320×200 PNG. +Выводы — для локального Compose, не production SLA. -## Ограничения и следующие шаги +## Ограничения -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Доставка **at-least-once**, не exactly-once: crash между publish и confirm/ack даёт дубликат; + эффект идемпотентен, но глобального exactly-once нет. +- Single-node RabbitMQ: durability при рестарте с volume, но не HA/replication. +- 40 Мп и 16-bit PNG не прогонялись в этой серии; worker имеет pixel cap, таймаут, watchdog + и memory limit, но пиковый RSS на 40 Мп нужно измерять отдельно. +- Track 2 — correctness harness (duplicate/stale/version + fault suite), не нагрузочный эксперимент; + interleavings sampled, не исчерпаны. +- Presign — optional extension; его ответ 202 и проверенный staging описаны в отчёте, но публичный + контракт его не требует. +- Rate limiting — admission cap с транзакционным reservation, не промышленный token bucket. +- Presign подписывает URL статическим локальным app-identity; S3 API опубликован только на + loopback для учебного стенда. В production нужен отдельный least-privilege signer и запрет + локального доступа к bucket. +- API readiness проверяет PostgreSQL, не S3/broker: при отказе S3 upload отвечает 503, но + маршрутизация LB и чтение метаданных продолжают работать. +- Reconciliation потерянной доставки запускается через `OUTBOX_RECONCILE_AFTER` (5m), поэтому + dead-letter не теряет intent, но восстановление не мгновенное. +- Failpoints включаются только `FAULTS_ENABLED=1` при `APP_ENV=dev`; в обычном стенде инертны. +- `x-delivery-limit` — immutable queue argument. На чистом стенде создаётся 100; при обновлении + существующего volume очередь нужно пересоздать (или применить policy) до старта worker/relay. +- Старые lab4 ready-вложения не переобрабатываются миграцией; для них нужен ручной + `jobctl reprocess` (backfill в эту серию не входил). + +## Трек 2: гипотеза, методика, результаты + +**Гипотеза.** При at-least-once доставке, effect key `(attachment_id, pipeline_version, variant)`, +deterministic S3 keys, serving-версии, меняющейся только после commit всех v2 variants, и ack +stale-доставок pipeline не теряет принятую работу и не перезаписывает новый результат старым. + +**Методика.** `scripts/lab5_track2.py --reps 3`: для каждой репетиции создаются новые пользователи/чат; +(1) worker остановлен, upload, две duplicate-публикации одного job, затем worker стартует; +(2) v1 ready → `jobctl reprocess -pipeline 2` → v2 ready → публикуется старое v1-сообщение. +Проверяются: уникальность variant rows, отсутствие регрессии ready, монотонность `pipeline_version`, +неизменность original, no-op stale job. Fault suite дополнительно проверяет crash окна и lease. + +**Результаты.** 6/6 сценариев PASS (`evidence/lab5/track2/runs.csv`), все fault-сценарии PASS. +Исходные данные — `evidence/lab5/track2/run.txt`, `summary.md`, `runs.csv`. + +**Почему не exactly-once.** Publisher confirm подтверждает приём брокером, consumer ack — устойчивый +эффект; crash между ними создаёт повторную доставку. Идемпотентный effect key и CAS делают повтор +безопасным, но не устраняют дубликаты доставки. + +## Ответы на вопросы защиты + +- **Publisher confirm vs consumer ack:** confirm — RabbitMQ durable accepted; ack — worker записал + эффект. 202 не ждёт ни того, ни другого. +- **Что сохранено к 202:** полный проверенный original в private S3 staging, attachment metadata + `queued`, job intent и outbox row в одной TX. Thumbnail/final original/broker publish могут ещё + не существовать. +- **Почему redelivery нормальна:** at-least-once; effect key, deterministic keys, CAS lease/attempt, + `ready`-no-op. Старый attempt/версия только ack-ается. +- **Stuck vs slow:** heartbeat продлевает lease; живой медленный worker не забирается. Мёртвый + перестаёт продлевать; reaper возвращает работу, старый CAS-fenced. +- **Replay:** только failed, явный CLI, audit events, budget attempt-ов; ready replay запрещён. diff --git a/cmd/jobctl/main.go b/cmd/jobctl/main.go new file mode 100644 index 00000000..248b8b33 --- /dev/null +++ b/cmd/jobctl/main.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "os" + "time" + + "github.com/google/uuid" + + "mtusi-messenger/internal/config" + "mtusi-messenger/internal/imageproc" + "mtusi-messenger/internal/logger" + "mtusi-messenger/internal/storage/postgres" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + cfg, err := config.Load() + if err != nil { + fmt.Fprintln(os.Stderr, "load config:", err) + os.Exit(1) + } + + log := logger.New(logger.WithLevel(cfg.SlogLevel()), logger.WithFormat(logger.FormatConsole)) + slog.SetDefault(log) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second) + if err != nil { + fmt.Fprintln(os.Stderr, "connect database:", err) + cancel() + os.Exit(1) //nolint:gocritic // main exits after DB setup failure + } + defer store.Close() + + repo := store.Pipeline() + + switch os.Args[1] { + case "failed": + runFailed(ctx, repo) + case "replay": + runReplay(ctx, repo, cfg) + case "reprocess": + runReprocess(ctx, repo, cfg) + case "duplicate": + runDuplicate(ctx, repo) + case "inspect": + runInspect(ctx, repo) + default: + usage() + os.Exit(2) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, `usage: jobctl [flags] + + failed -limit N + replay -job JOB_ID [-attempts N] + reprocess -attachment ATTACHMENT_ID -pipeline N + duplicate -attachment ATTACHMENT_ID | -job JOB_ID + inspect -attachment ATTACHMENT_ID`) +} + +func runFailed(ctx context.Context, repo *postgres.PipelineRepository) { + flags := flag.NewFlagSet("failed", flag.ExitOnError) + limit := flags.Int("limit", 50, "maximum jobs") + _ = flags.Parse(os.Args[2:]) + + jobs, err := repo.ListFailedJobs(ctx, *limit) + if err != nil { + fail(err) + } + + printJSON(jobs) +} + +func runReplay(ctx context.Context, repo *postgres.PipelineRepository, cfg config.Config) { + flags := flag.NewFlagSet("replay", flag.ExitOnError) + jobID := flags.String("job", "", "job id") + attempts := flags.Int("attempts", cfg.Processing.MaxAttempts, "additional attempts") + _ = flags.Parse(os.Args[2:]) + + id, err := uuid.Parse(*jobID) + if err != nil { + fail(fmt.Errorf("invalid job id: %w", err)) + } + + job, err := repo.ReplayJob(ctx, id, *attempts) + if err != nil { + fail(err) + } + + printJSON(job) +} + +func runReprocess(ctx context.Context, repo *postgres.PipelineRepository, cfg config.Config) { + flags := flag.NewFlagSet("reprocess", flag.ExitOnError) + attachmentID := flags.String("attachment", "", "attachment id") + version := flags.Int("pipeline", 2, "target pipeline version") + _ = flags.Parse(os.Args[2:]) + + if _, ok := imageproc.Pipeline(*version); !ok { + fail(fmt.Errorf("unsupported pipeline version %d", *version)) + } + + id, err := uuid.Parse(*attachmentID) + if err != nil { + fail(fmt.Errorf("invalid attachment id: %w", err)) + } + + jobID, err := uuid.NewV7() + if err != nil { + fail(err) + } + + job, err := repo.Reprocess(ctx, id, *version, cfg.Processing.MaxAttempts, jobID) + if err != nil { + fail(err) + } + + printJSON(job) +} + +func runDuplicate(ctx context.Context, repo *postgres.PipelineRepository) { + flags := flag.NewFlagSet("duplicate", flag.ExitOnError) + attachmentID := flags.String("attachment", "", "attachment id") + jobID := flags.String("job", "", "specific job id (old pipeline version)") + _ = flags.Parse(os.Args[2:]) + + if *jobID != "" { + id, err := uuid.Parse(*jobID) + if err != nil { + fail(fmt.Errorf("invalid job id: %w", err)) + } + + message, err := repo.DuplicateJob(ctx, id) + if err != nil { + fail(err) + } + + printJSON(message) + + return + } + + id, err := uuid.Parse(*attachmentID) + if err != nil { + fail(fmt.Errorf("attachment or job id is required: %w", err)) + } + + message, err := repo.Duplicate(ctx, id) + if err != nil { + fail(err) + } + + printJSON(message) +} + +func runInspect(ctx context.Context, repo *postgres.PipelineRepository) { + flags := flag.NewFlagSet("inspect", flag.ExitOnError) + attachmentID := flags.String("attachment", "", "attachment id") + _ = flags.Parse(os.Args[2:]) + + id, err := uuid.Parse(*attachmentID) + if err != nil { + fail(fmt.Errorf("invalid attachment id: %w", err)) + } + + attachment, err := repo.GetForID(ctx, id) + if err != nil { + fail(err) + } + + variants, err := repo.ListVariants(ctx, id, attachment.PipelineVersion) + if err != nil { + fail(err) + } + + var jobOutput any + + job, jobErr := repo.GetJobByAttachment(ctx, id, attachment.PipelineVersion) + if jobErr == nil { + jobOutput = job + } + + printJSON(map[string]any{ + "attachment": attachment, + "variants": variants, + "job": jobOutput, + }) +} + +func printJSON(value any) { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(value); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 841173ed..45c46e95 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -74,23 +74,31 @@ func run(cfg config.Config, log *slog.Logger) error { return fmt.Errorf("connect object store: %w", err) } - attachmentRepository := store.Attachments() + pipelineRepository := store.Pipeline() attachmentService := attachment.NewService( - attachmentRepository, + pipelineRepository, objectStore, attachment.Config{ - MaxUploadBytes: cfg.S3.MaxUploadBytes, - MaxImagePixels: cfg.S3.MaxImagePixels, - PresignTTL: cfg.S3.PresignTTL, + MaxUploadBytes: cfg.S3.MaxUploadBytes, + MaxImagePixels: cfg.S3.MaxImagePixels, + PresignTTL: cfg.S3.PresignTTL, + PipelineVersion: cfg.Processing.PipelineVersion, + MaxAttempts: cfg.Processing.MaxAttempts, + MaxPendingPerUser: cfg.Admission.MaxPendingPerUser, + MaxPending: cfg.Admission.MaxPending, + MaxStagedBytes: cfg.Admission.MaxStagedBytes, + MaxConcurrent: cfg.Admission.MaxConcurrent, }, log, ) sweeper := attachment.NewSweeper( - attachmentRepository, + pipelineRepository, objectStore, store, cfg.S3.SweepGrace, + cfg.Staging.OrphanGrace, + cfg.Staging.FailedRetention, cfg.S3.SweepInterval, log, ) @@ -151,7 +159,7 @@ func run(cfg config.Config, log *slog.Logger) error { rest.NewHandler( user.NewService(store.Users()), chat.NewService(chatRepo), - message.NewService(store.Messages(), attachmentRepository), + message.NewService(store.Messages(), pipelineRepository), attachmentService, health.New(pingers...), sessionService, diff --git a/cmd/relay/main.go b/cmd/relay/main.go new file mode 100644 index 00000000..402be971 --- /dev/null +++ b/cmd/relay/main.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "mtusi-messenger/internal/broker" + "mtusi-messenger/internal/config" + "mtusi-messenger/internal/domain" + "mtusi-messenger/internal/logger" + "mtusi-messenger/internal/outbox" + "mtusi-messenger/internal/storage/postgres" +) + +const maxUnpublishedAge = 10 * time.Minute + +func main() { + log := logger.New() + + cfg, err := config.Load() + if err != nil { + log.Error("load config", slog.Any("error", err)) + os.Exit(1) + } + + log = logger.New( + logger.WithLevel(cfg.SlogLevel()), + logger.WithFormat(cfg.LoggerFormat()), + ) + slog.SetDefault(log) + + if err := run(cfg, log); err != nil { + log.Error("shutdown with error", slog.Any("error", err)) + os.Exit(1) + } +} + +type publisher struct { + cfg broker.Config + log *slog.Logger + mu sync.Mutex + conn *broker.Client +} + +func (p *publisher) client() (*broker.Client, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.conn != nil && p.conn.Connected() { + return p.conn, nil + } + + if p.conn != nil { + _ = p.conn.Close() + p.conn = nil + } + + conn, err := broker.Dial(p.cfg) + if err != nil { + return nil, err + } + + p.conn = conn + + return conn, nil +} + +func (p *publisher) PublishConfirmed( + ctx context.Context, + routingKey string, + message domain.JobMessage, +) error { + conn, err := p.client() + if err != nil { + return err + } + + if err := conn.PublishConfirmed(ctx, routingKey, message); err != nil { + p.mu.Lock() + if p.conn == conn { + _ = p.conn.Close() + p.conn = nil + } + p.mu.Unlock() + + return err + } + + return nil +} + +func (p *publisher) Connect() error { + _, err := p.client() + + return err +} + +func (p *publisher) Connected() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.conn != nil && p.conn.Connected() +} + +func (p *publisher) Close() { + p.mu.Lock() + defer p.mu.Unlock() + + if p.conn != nil { + _ = p.conn.Close() + p.conn = nil + } +} + +func brokerConfig(cfg config.Config) broker.Config { + return broker.Config{ + URL: cfg.Broker.URL, + Exchange: cfg.Broker.Exchange, + DLX: cfg.Broker.DLX, + Queue: cfg.Broker.Queue, + DLQ: cfg.Broker.DLQ, + ConnectTimeout: cfg.Broker.ConnectTimeout, + ConfirmTimeout: cfg.Broker.ConfirmTimeout, + Heartbeat: cfg.Broker.Heartbeat, + Prefetch: cfg.Broker.Prefetch, + } +} + +func run(cfg config.Config, log *slog.Logger) error { + if cfg.Broker.URL == "" { + return errors.New("BROKER_URL is required for relay") + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second) + if err != nil { + return fmt.Errorf("connect database: %w", err) + } + defer store.Close() + + pub := &publisher{cfg: brokerConfig(cfg), log: log} + defer pub.Close() + + if err := pub.Connect(); err != nil { + log.Warn("broker is not reachable yet; relay will retry", slog.Any("error", err)) + } + + relay := outbox.New(store.Pipeline(), pub, outbox.Config{ + Owner: "relay-" + cfg.Broker.Queue, + PollInterval: cfg.Outbox.PollInterval, + ClaimTTL: cfg.Outbox.ClaimTTL, + Batch: cfg.Outbox.Batch, + ReaperInterval: cfg.Outbox.ReaperInterval, + ReconcileAfter: cfg.Outbox.ReconcileAfter, + Retention: cfg.Outbox.Retention, + }, log) + + relayCtx, cancelRelay := context.WithCancel(ctx) + defer cancelRelay() + + relayDone := make(chan struct{}) + + go func() { + defer close(relayDone) + + relay.Run(relayCtx) + }() + + server := healthServer(cfg.Relay.HealthAddr, store, pub, relay, log) + + go func() { + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("health server failed", slog.Any("error", err)) + } + }() + + <-ctx.Done() + cancelRelay() + <-relayDone + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return server.Shutdown(shutdownCtx) +} + +func healthServer( + addr string, + store *postgres.Store, + pub *publisher, + relay *outbox.Relay, + _ *slog.Logger, +) *http.Server { + mux := http.NewServeMux() + + mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { + writeHealth(w, http.StatusOK, map[string]any{"status": "ok"}) + }) + + mux.HandleFunc("/health/ready", func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + + if err := store.Ping(ctx); err != nil { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"}) + + return + } + + if !pub.Connected() { + if err := pub.Connect(); err != nil { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"}) + + return + } + } + + oldest, err := relay.Health(ctx) + if err != nil { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"}) + + return + } + + if oldest > maxUnpublishedAge { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{ + "status": "unavailable", + "outbox_oldest_seconds": int(oldest.Seconds()), + }) + + return + } + + writeHealth(w, http.StatusOK, map[string]any{ + "status": "ok", + "outbox_oldest_seconds": int(oldest.Seconds()), + }) + }) + + return &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } +} + +func writeHealth(w http.ResponseWriter, status int, body map[string]any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..d61bf4ab --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,231 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "mtusi-messenger/internal/broker" + "mtusi-messenger/internal/config" + "mtusi-messenger/internal/logger" + "mtusi-messenger/internal/platform/objectstore" + "mtusi-messenger/internal/storage/postgres" + "mtusi-messenger/internal/worker" +) + +func main() { + log := logger.New() + + cfg, err := config.Load() + if err != nil { + log.Error("load config", slog.Any("error", err)) + os.Exit(1) + } + + log = logger.New( + logger.WithLevel(cfg.SlogLevel()), + logger.WithFormat(cfg.LoggerFormat()), + ) + slog.SetDefault(log) + + if err := run(cfg, log); err != nil { + log.Error("shutdown with error", slog.Any("error", err)) + os.Exit(1) + } +} + +func run(cfg config.Config, log *slog.Logger) error { + if cfg.Broker.URL == "" { + return errors.New("BROKER_URL is required for worker") + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second) + if err != nil { + return fmt.Errorf("connect database: %w", err) + } + defer store.Close() + + objectStore, err := objectstore.New(objectstore.Config{ + Endpoint: cfg.S3.Endpoint, + PublicEndpoint: cfg.S3.PublicEndpoint, + Region: cfg.S3.Region, + Bucket: cfg.S3.Bucket, + AccessKeyID: cfg.S3.AccessKeyID, + SecretAccessKey: cfg.S3.SecretAccessKey, + UsePathStyle: cfg.S3.UsePathStyle, + Timeout: cfg.S3.Timeout, + }) + if err != nil { + return fmt.Errorf("connect object store: %w", err) + } + + workerInstance := worker.New(store.Pipeline(), objectStore, worker.Config{ + Concurrency: cfg.Worker.Concurrency, + LeaseTTL: cfg.Processing.LeaseTTL, + Heartbeat: cfg.Processing.Heartbeat, + JobTimeout: cfg.Processing.JobTimeout, + RetryBase: cfg.Processing.RetryBase, + RetryCap: cfg.Processing.RetryCap, + MaxUploadBytes: cfg.S3.MaxUploadBytes, + MaxImagePixels: cfg.S3.MaxImagePixels, + }, log) + + server := healthServer(cfg.Worker.HealthAddr, store, objectStore, workerInstance) + + var serverWait sync.WaitGroup + serverWait.Add(1) + + go func() { + defer serverWait.Done() + + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("health server failed", slog.Any("error", err)) + } + }() + + runCtx, cancelRun := context.WithCancel(ctx) + defer cancelRun() + + consumerDone := make(chan struct{}) + + go func() { + defer close(consumerDone) + + consumeWithReconnect(runCtx, cfg, workerInstance, log) + }() + + <-ctx.Done() + cancelRun() + <-consumerDone + + shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Worker.ShutdownTimeout) + defer cancel() + + _ = server.Shutdown(shutdownCtx) + serverWait.Wait() + + return nil +} + +func consumeWithReconnect( + ctx context.Context, + cfg config.Config, + workerInstance *worker.Worker, + log *slog.Logger, +) { + for ctx.Err() == nil { + client, err := broker.Dial(brokerConfig(cfg)) + if err != nil { + log.WarnContext(ctx, "broker dial failed, retrying", slog.Any("error", err)) + + if !sleepCtx(ctx, 2*time.Second) { + return + } + + continue + } + + err = workerInstance.Run(ctx, client) + _ = client.Close() + + if ctx.Err() != nil { + return + } + + if err != nil { + log.WarnContext(ctx, "consumer stopped, reconnecting", slog.Any("error", err)) + } + + if !sleepCtx(ctx, 2*time.Second) { + return + } + } +} + +func sleepCtx(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func brokerConfig(cfg config.Config) broker.Config { + return broker.Config{ + URL: cfg.Broker.URL, + Exchange: cfg.Broker.Exchange, + DLX: cfg.Broker.DLX, + Queue: cfg.Broker.Queue, + DLQ: cfg.Broker.DLQ, + ConnectTimeout: cfg.Broker.ConnectTimeout, + ConfirmTimeout: cfg.Broker.ConfirmTimeout, + Heartbeat: cfg.Broker.Heartbeat, + Prefetch: cfg.Worker.Prefetch, + } +} + +func healthServer( + addr string, + store *postgres.Store, + objectStore objectstore.Store, + workerInstance *worker.Worker, +) *http.Server { + mux := http.NewServeMux() + + mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { + writeHealth(w, http.StatusOK, map[string]any{"status": "ok"}) + }) + + mux.HandleFunc("/health/ready", func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + if err := store.Ping(ctx); err != nil { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"}) + + return + } + + if err := objectStore.Ping(ctx); err != nil { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"}) + + return + } + + running, connected := workerInstance.Ready() + if !running || !connected { + writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"}) + + return + } + + writeHealth(w, http.StatusOK, map[string]any{"status": "ok"}) + }) + + return &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } +} + +func writeHealth(w http.ResponseWriter, status int, body map[string]any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} diff --git a/compose.yaml b/compose.yaml index 8d1a3adf..03075f3c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,5 @@ -# Lab4 stand: two API replicas behind HAProxy, PostgreSQL, Redis and SeaweedFS S3. +# Lab5 stand: API replicas behind HAProxy, PostgreSQL, Redis, SeaweedFS S3, +# a durable RabbitMQ broker, one outbox relay and a separate image worker. # # cp -n .env.example .env # docker compose --env-file .env up -d --build @@ -8,13 +9,13 @@ # reachable from the host via `docker compose exec` / `docker compose logs`. # No container_name anywhere: replica hostnames stay the short container IDs, # which the application reports back as X-Instance-Id. -name: mtusi-messenger-lab4 +name: mtusi-messenger-lab5 # The image is built once by the api service. migrate and s3-init only consume # it; duplicating `build` here makes Compose compile the same Go tree in parallel # and OOMs Docker Desktop's default VM. x-app: &app - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 services: postgres: @@ -42,7 +43,7 @@ services: restart: unless-stopped redis: - image: redis:7.4-alpine + image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 command: - redis-server - --appendonly @@ -100,6 +101,35 @@ services: memory: 640M restart: unless-stopped + + # Durable job transport. Quorum queues, publisher confirms and a DLQ live in + # the broker; PostgreSQL's outbox remains the source of truth for intent. + broker: + image: rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 + hostname: broker + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-messenger} + RABBITMQ_DEFAULT_VHOST: ${BROKER_VHOST:-messenger} + volumes: + - rabbitmq-data:/var/lib/rabbitmq + ports: + # Management UI only on loopback, for operator evidence. AMQP stays internal. + - "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + deploy: + resources: + limits: + cpus: "${BROKER_CPU_LIMIT:-0.50}" + memory: ${BROKER_MEM_LIMIT:-384M} + stop_grace_period: 30s + restart: unless-stopped + # One-shot, idempotent bucket bootstrap and identity probe. s3-init: <<: *app @@ -184,6 +214,19 @@ services: S3_SWEEP_GRACE: ${S3_SWEEP_GRACE:-10m} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1} + PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5} + PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s} + PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s} + PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s} + PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s} + PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s} + STAGING_ORPHAN_GRACE: ${STAGING_ORPHAN_GRACE:-1h} + STAGING_FAILED_RETENTION: ${STAGING_FAILED_RETENTION:-168h} + ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20} + ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200} + ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648} + ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8} # Размер пула соединений ОДНОЙ реплики; общий расход = реплики × значение. # Это один из общих ресурсов, который масштабирование расходует первым. POSTGRES_MAX_CONNS: "${POSTGRES_MAX_CONNS:-10}" @@ -217,10 +260,133 @@ services: start_period: 5s restart: unless-stopped + + # Separate worker process. It is intentionally not published and can be + # stopped/scaled independently from the API; stopping it must not stop 202. + worker: + <<: *app + entrypoint: ["/usr/local/bin/worker"] + env_file: + - path: .env + required: false + environment: + APP_CONFIG: /etc/messenger/dev/config.yaml + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger} + POSTGRES_SSL_MODE: disable + S3_ENDPOINT: http://seaweedfs:8333 + S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret} + S3_USE_PATH_STYLE: "true" + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + WORKER_SHUTDOWN_TIMEOUT: ${WORKER_SHUTDOWN_TIMEOUT:-30s} + WORKER_HEALTH_ADDR: ":8081" + FAULTS_ENABLED: ${FAULTS_ENABLED:-0} + FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-} + FAULT_WORKER_AFTER_CLAIM: ${FAULT_WORKER_AFTER_CLAIM:-} + FAULT_WORKER_AFTER_FIRST_VARIANT: ${FAULT_WORKER_AFTER_FIRST_VARIANT:-} + FAULT_WORKER_AFTER_PUT: ${FAULT_WORKER_AFTER_PUT:-} + FAULT_WORKER_AFTER_COMMIT: ${FAULT_WORKER_AFTER_COMMIT:-} + FAULT_WORKER_FAIL: ${FAULT_WORKER_FAIL:-} + FAULT_WORKER_SLEEP_AFTER_CLAIM: ${FAULT_WORKER_SLEEP_AFTER_CLAIM:-} + PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1} + PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5} + PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s} + PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s} + PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s} + PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s} + PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s} + ports: + - "127.0.0.1:${WORKER_HEALTH_PORT:-8081}:8081" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + deploy: + resources: + limits: + cpus: "${WORKER_CPU_LIMIT:-1.00}" + memory: ${WORKER_MEM_LIMIT:-1G} + stop_grace_period: 40s + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8081/health/ready"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + restart: unless-stopped + + # Outbox publisher + lease reaper. One replica: duplicate relays are safe but + # a single owner keeps the evidence and the claim reaper simple. + relay: + <<: *app + entrypoint: ["/usr/local/bin/relay"] + env_file: + - path: .env + required: false + environment: + APP_CONFIG: /etc/messenger/dev/config.yaml + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger} + POSTGRES_SSL_MODE: disable + BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger} + BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1} + BROKER_DLQ: ${BROKER_DLQ:-attachment.process.dead} + OUTBOX_POLL_INTERVAL: ${OUTBOX_POLL_INTERVAL:-1s} + OUTBOX_CLAIM_TTL: ${OUTBOX_CLAIM_TTL:-30s} + OUTBOX_BATCH: ${OUTBOX_BATCH:-20} + OUTBOX_REAPER_INTERVAL: ${OUTBOX_REAPER_INTERVAL:-15s} + OUTBOX_RECONCILE_AFTER: ${OUTBOX_RECONCILE_AFTER:-5m} + OUTBOX_RETENTION: ${OUTBOX_RETENTION:-168h} + RELAY_HEALTH_ADDR: ":8082" + FAULTS_ENABLED: ${FAULTS_ENABLED:-0} + FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-} + FAULT_OUTBOX_AFTER_COMMIT: ${FAULT_OUTBOX_AFTER_COMMIT:-} + ports: + - "127.0.0.1:${RELAY_HEALTH_PORT:-8082}:8082" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + deploy: + replicas: 1 + resources: + limits: + cpus: "${RELAY_CPU_LIMIT:-0.25}" + memory: ${RELAY_MEM_LIMIT:-128M} + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/health/ready"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + restart: unless-stopped + # Public entry point. Active /health/ready probes remove a not-ready replica from # rotation entirely; infra/lb/haproxy.cfg holds the upstream, checks and retries. lb: - image: haproxy:3.2-alpine + image: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 volumes: - ./infra/lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro ports: @@ -241,3 +407,4 @@ volumes: postgres18-data: redis7-data: seaweedfs-data: + rabbitmq-data: diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index f61ef202..ec17c1aa 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -1,8 +1,8 @@ -name: mtusi-messenger-lab4 +name: mtusi-messenger-lab5 # Built once by the api service; migrate and s3-init consume the same image. x-app: &app - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 services: postgres: @@ -23,7 +23,7 @@ services: start_period: 5s restart: unless-stopped redis: - image: redis:7.4-alpine + image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 command: - redis-server - --appendonly @@ -65,6 +65,26 @@ services: start_period: 60s restart: unless-stopped + + broker: + image: rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 + hostname: broker + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-messenger} + RABBITMQ_DEFAULT_VHOST: ${BROKER_VHOST:-messenger} + volumes: + - rabbitmq-data:/var/lib/rabbitmq + ports: + - "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + restart: unless-stopped + s3-init: <<: *app entrypoint: ["/usr/local/bin/s3init"] @@ -129,6 +149,19 @@ services: S3_USE_PATH_STYLE: "true" MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1} + PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5} + PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s} + PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s} + PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s} + PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s} + PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s} + STAGING_ORPHAN_GRACE: ${STAGING_ORPHAN_GRACE:-1h} + STAGING_FAILED_RETENTION: ${STAGING_FAILED_RETENTION:-168h} + ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20} + ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200} + ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648} + ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8} ports: - "127.0.0.1:${SERVER_PORT:-8080}:8080" depends_on: @@ -148,10 +181,103 @@ services: start_period: 5s restart: unless-stopped + + worker: + <<: *app + entrypoint: ["/usr/local/bin/worker"] + env_file: + - path: .env + required: false + environment: + APP_CONFIG: /etc/messenger/dev/config.yaml + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger} + POSTGRES_SSL_MODE: disable + S3_ENDPOINT: http://seaweedfs:8333 + S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret} + S3_USE_PATH_STYLE: "true" + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + WORKER_HEALTH_ADDR: ":8081" + FAULTS_ENABLED: ${FAULTS_ENABLED:-0} + FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-} + PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1} + PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5} + PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s} + PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s} + PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s} + PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s} + PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s} + ports: + - "127.0.0.1:${WORKER_HEALTH_PORT:-8081}:8081" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8081/health/ready"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + restart: unless-stopped + + relay: + <<: *app + entrypoint: ["/usr/local/bin/relay"] + env_file: + - path: .env + required: false + environment: + APP_CONFIG: /etc/messenger/dev/config.yaml + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger} + POSTGRES_SSL_MODE: disable + BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger} + RELAY_HEALTH_ADDR: ":8082" + FAULTS_ENABLED: ${FAULTS_ENABLED:-0} + FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-} + ports: + - "127.0.0.1:${RELAY_HEALTH_PORT:-8082}:8082" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/health/ready"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + restart: unless-stopped + volumes: postgres18-data: - name: mtusi-messenger-lab4-postgres18-data + name: mtusi-messenger-lab5-postgres18-data redis7-data: - name: mtusi-messenger-lab4-redis7-data + name: mtusi-messenger-lab5-redis7-data seaweedfs-data: - name: mtusi-messenger-lab4-seaweedfs-data + name: mtusi-messenger-lab5-seaweedfs-data + rabbitmq-data: + name: mtusi-messenger-lab5-rabbitmq-data diff --git a/evidence/lab5/README.md b/evidence/lab5/README.md new file mode 100644 index 00000000..3411caea --- /dev/null +++ b/evidence/lab5/README.md @@ -0,0 +1,20 @@ +# Lab5 evidence + +All artifacts were produced against the local Compose stand on 2026-10-30 +(2 vCPU, 2 GiB Docker VM, RabbitMQ 4.3.7, PostgreSQL 18, SeaweedFS 3.80). + +- `stand.txt` — containers, image versions, VM limits. +- `smoke.txt` — public `make test`. +- `async.txt` — worker down/restart, broker restart with volume, stack restart. +- `variants.txt` — thumbnail/crop/optimized metadata, bytes, dimensions. +- `faults.txt` — permanent/transient retry, DLQ/replay, crash before/after + side effect, crash before/after ack, outbox gap, lease takeover. +- `persistence.txt`, `failover.txt`, `idempotency.txt`, `lb-readiness.txt` — + cumulative lab2–lab4 regressions. +- `track2/run.txt`, `track2/runs.csv`, `track2/summary.md` — track 2 + duplicate/stale/version harness (3 repetitions). +- `unit-test.txt`, `integration-test.txt`, `vet.txt`, `lint.txt`, + `validate.txt` — build/quality gates. + +No `.env`, tokens, credentials or presigned URLs are stored here. Raw SQL rows +contain only synthetic IDs and metadata. diff --git a/evidence/lab5/async.txt b/evidence/lab5/async.txt new file mode 100644 index 00000000..7e1a7910 --- /dev/null +++ b/evidence/lab5/async.txt @@ -0,0 +1,12 @@ +# make test-queue 2026-10-31T11:26:40Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/lab5_async.py --base-url "http://localhost:8080" +worker down: 202 queued, content 409, queued attachment linked, API restarted +worker restarted: ready, thumbnail and original downloadable, staging cleared +broker down: upload accepted into outbox +broker + relay recovered: queued job reached ready +stack restart: ready attachment and original preserved +worker down: reprocess pipeline 2 accepted, v1 audit event recorded, upload still queued +worker up: reprocessed upload reached ready at pipeline 2, v1 superseded +worker down: reprocess pipeline 3 refused, v1 job untouched +worker up: preserved upload reached ready at pipeline 1 +PASS: lab5 async worker/broker durability diff --git a/evidence/lab5/attachments-negative.txt b/evidence/lab5/attachments-negative.txt new file mode 100644 index 00000000..a2a255de --- /dev/null +++ b/evidence/lab5/attachments-negative.txt @@ -0,0 +1,25 @@ +# make test-attachments 2026-10-31T11:26:14Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/attachments_negative.py --base-url "http://localhost:8080" +10 MiB + 1 body: 413 +empty body: 415 +corrupt PNG (IDAT CRC): 415 +MIME spoof (PNG declared as JPEG): 415 +structurally valid PNG with non-zlib IDAT: 415 +valid zlib, empty PNG raster: 415 +valid zlib, invalid PNG scanline filter: 415 +invalid PNG color type/bit depth pair: 415 +duplicate PNG IHDR: 415 +trailing bytes after PNG zlib stream: 415 +tRNS on RGBA PNG: 415 +short tRNS on RGB PNG: 415 +PLTE entries exceed palette bit depth: 415 +palette tRNS exceeds PLTE entries: 415 +dangerous filenames: 400 +foreign attachment metadata/content: 404 +empty text without attachments: 400 +attachment from another chat: 404 +11 attachment ids: 400 +duplicate attachment ids: 400 +attachment-only message: 201 +anonymous direct S3 GET: 403 +PASS: lab4 attachment negative scenarios diff --git a/evidence/lab5/db-state.txt b/evidence/lab5/db-state.txt new file mode 100644 index 00000000..c1126aa4 --- /dev/null +++ b/evidence/lab5/db-state.txt @@ -0,0 +1,29 @@ +# db state 2026-10-30T20:16:08Z + status | count +---------+------- + failed | 2 + pending | 4 + ready | 31 +(3 rows) + + status | count +--------+------- + failed | 2 + ready | 34 +(2 rows) + + variants +---------- + 102 +(1 row) + + unpublished | outbox +-------------+-------- + 0 | 58 +(1 row) + +Timeout: 60.0 seconds ... +Listing queues for vhost messenger ... +name messages messages_ready messages_unacknowledged consumers +attachment.process.dead 4 4 0 0 +attachment.process.v1 0 0 0 0 diff --git a/evidence/lab5/failover.txt b/evidence/lab5/failover.txt new file mode 100644 index 00000000..2ec4623e --- /dev/null +++ b/evidence/lab5/failover.txt @@ -0,0 +1,5 @@ +# failover 2026-10-30T20:11:43Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 0, "first_recovery_seconds": 0.221} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5/faults.txt b/evidence/lab5/faults.txt new file mode 100644 index 00000000..4be57ec9 --- /dev/null +++ b/evidence/lab5/faults.txt @@ -0,0 +1,15 @@ +# make test-faults 2026-10-31T11:27:03Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/lab5_faults.py --base-url "http://localhost:8080" +permanent failure: failed/injected_failure, content 409, failed attachment not linkable, registry visible +replay: failed -> queued -> ready, one thumbnail +replay ready job: refused +replay older failed version while v2 queued: refused, state intact +v2 published after older replay refusal +replay of failed reprocess: queued projection restored, then ready at pipeline 2 +transient exhaustion: attempts=5, retry_exhausted, four retries, DLQ message present +crash after first variant: redelivered, same three variants, attempts=2 +crash after commit before ack: ready remains, redelivery is a no-op +crash after claim before side effect: requeued, attempts=2, ready +outbox gap: crash before publish recovered through the relay, outbox published, ready +lease expiry: paused worker, reaper requeued, stale attempt fenced, ready +PASS: lab5 level 4 fault injection diff --git a/evidence/lab5/idempotency.txt b/evidence/lab5/idempotency.txt new file mode 100644 index 00000000..03f99f44 --- /dev/null +++ b/evidence/lab5/idempotency.txt @@ -0,0 +1,4 @@ +# idempotency 2026-10-30T20:11:43Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/idempotency_parallel.py --base-url "http://localhost:8080" +{"replay": {"workers": 12, "instances": ["1c67613c89f2", "ee96975adb44"], "message_id": "01a18e70-8fcd-7519-91ab-d9c43b6a5f37"}, "conflict": {"workers": 12, "accepted": 1, "conflicts": 11}, "stable_body": {"message_id": "01a18e70-8ff9-759f-ab87-abd58f58c59d"}, "key_required": {"status": 400}} +PASS: параллельный повтор с одним Idempotency-Key создаёт ровно одно сообщение diff --git a/evidence/lab5/integration-test.txt b/evidence/lab5/integration-test.txt new file mode 100644 index 00000000..08905b2b --- /dev/null +++ b/evidence/lab5/integration-test.txt @@ -0,0 +1,2 @@ +ok mtusi-messenger/internal/storage/postgres 9.448s +ok mtusi-messenger/internal/storage/redis 1.238s diff --git a/evidence/lab5/lb-readiness-detail.txt b/evidence/lab5/lb-readiness-detail.txt new file mode 100644 index 00000000..eeb57ae3 --- /dev/null +++ b/evidence/lab5/lb-readiness-detail.txt @@ -0,0 +1,31 @@ +# LB excludes a not-ready but reachable replica (lab5) +# generated: 2026-10-30T20:18:32Z +# lb: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 + +## 1. baseline: requests through LB by replica + 22 23d07355bd77 + 18 56bd7451c245 + stats: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + +## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias + instance-id: 42c73eb4c0a5, readiness while its database is reachable: 200 + load balancer slot for the candidate: api2, stats: api1=UP api2=UP 1/2 api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + +## 3. cut the candidate's database access (stop only the TCP proxy) + candidate is still running: running, readiness: 503 + waiting for the load balancer to mark api2 DOWN + api2 went DOWN 1s after its database was cut (check: inter 2s fall 2) + stats after failure: api1=UP api2=DOWN api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + +## 4. 40 requests through LB while the candidate is not ready + 22 23d07355bd77 + 18 56bd7451c245 + (the candidate must not appear above) + +## 5. business writes through LB during the exclusion + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 816.58, + +## 6. cleanup + stats after cleanup: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP diff --git a/evidence/lab5/lb-readiness.txt b/evidence/lab5/lb-readiness.txt new file mode 100644 index 00000000..1ec95381 --- /dev/null +++ b/evidence/lab5/lb-readiness.txt @@ -0,0 +1,34 @@ +# lb readiness 2026-10-30T20:18:31Z +scripts/lb_readiness_check.sh +# LB excludes a not-ready but reachable replica (lab5) +# generated: 2026-10-30T20:18:32Z +# lb: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 + +## 1. baseline: requests through LB by replica + 22 23d07355bd77 + 18 56bd7451c245 + stats: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + +## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias + instance-id: 42c73eb4c0a5, readiness while its database is reachable: 200 + load balancer slot for the candidate: api2, stats: api1=UP api2=UP 1/2 api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + +## 3. cut the candidate's database access (stop only the TCP proxy) + candidate is still running: running, readiness: 503 + waiting for the load balancer to mark api2 DOWN + api2 went DOWN 1s after its database was cut (check: inter 2s fall 2) + stats after failure: api1=UP api2=DOWN api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + +## 4. 40 requests through LB while the candidate is not ready + 22 23d07355bd77 + 18 56bd7451c245 + (the candidate must not appear above) + +## 5. business writes through LB during the exclusion + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 816.58, + +## 6. cleanup + stats after cleanup: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP +lb_readiness_check.sh: evidence in evidence/lab5/lb-readiness-detail.txt diff --git a/evidence/lab5/lint.txt b/evidence/lab5/lint.txt new file mode 100644 index 00000000..8197453a --- /dev/null +++ b/evidence/lab5/lint.txt @@ -0,0 +1,20 @@ +/Users/vladislav/Projects/mtusi2026lab5/bin/golangci-lint run -v +level=info msg="golangci-lint has version 2.13.2 built with go1.27.1 from (unknown, modified: ?, mod sum: \"h1:bCyq3E4vo9qwzifjpzJqYndEt7Ncva80qkk8G2B4TXU=\") on (unknown)" +level=info msg="[config_reader] Config search paths: [./ /Users/vladislav/Projects/mtusi2026lab5 /Users/vladislav/Projects /Users/vladislav /Users /]" +level=info msg="[config_reader] Used config file .golangci.yml" +level=info msg="[config_reader] Module name \"mtusi-messenger\"" +level=info msg="[goenv] Read go env for 6.264542ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}" +level=info msg="[lintersdb] Active 50 linters: [asasalint bodyclose canonicalheader containedctx contextcheck copyloopvar depguard durationcheck errcheck errname errorlint exhaustive exptostd fatcontext forbidigo forcetypeassert gocheckcompilerdirectives gocritic gofumpt goimports gomoddirectives gosec govet ineffassign intrange makezero misspell musttag nilerr nilnil noctx nolintlint perfsprint prealloc reassign recvcheck revive rowserrcheck sloglint sqlclosecheck staticcheck testifylint thelper tparallel unconvert unparam unused usestdlibvars usetesting wastedassign]" +level=info msg="[loader] Using build tags: [integration]" +level=info msg="[loader] Go packages loading at mode 8767 (compiled_files|deps|exports_file|files|types_sizes|imports|name) took 193.808375ms" +level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 11.667ms" +level=info msg="[linters_context/goanalysis] analyzers took 0s with no stages" +level=info msg="[runner/exclusion_rules] Skipped 14 issues by rules: [Path: \"_test\\\\.go\", Linters: \"gosec, prealloc, forcetypeassert, unparam\"]" +level=info msg="[runner] Issues before processing: 887, after processing: 0" +level=info msg="[runner] Processors filtering stat (in/out): path_absoluter: 887/887, cgo: 887/887, filename_unadjuster: 887/887, nolint_filter: 6/0, invalid_issue: 887/887, exclusion_paths: 887/887, path_relativity: 887/887, generated_file_filter: 887/298, exclusion_rules: 298/6" +level=info msg="[runner] processing took 2.828168ms with stages: generated_file_filter: 1.490334ms, nolint_filter: 764.209µs, path_relativity: 284.917µs, exclusion_rules: 251.001µs, invalid_issue: 13.417µs, cgo: 12.25µs, path_absoluter: 4.666µs, sort_results: 2.916µs, filename_unadjuster: 2.542µs, max_same_issues: 874ns, diff: 167ns, fixer: 166ns, path_prettifier: 125ns, max_per_file_from_linter: 125ns, exclusion_paths: 125ns, path_shortener: 124ns, uniq_by_line: 84ns, severity-rules: 42ns, max_from_linter: 42ns, source_code: 42ns" +level=info msg="[runner] linters took 629.148708ms with stages: goanalysis_metalinter: 626.277292ms" +0 issues. +level=info msg="File cache stats: 0 entries of total size 0B" +level=info msg="Memory: 10 samples, avg is 87.2MB, max is 111.5MB" +level=info msg="Execution took 841.622625ms" diff --git a/evidence/lab5/persistence.txt b/evidence/lab5/persistence.txt new file mode 100644 index 00000000..933a8846 --- /dev/null +++ b/evidence/lab5/persistence.txt @@ -0,0 +1,4 @@ +# persistence 2026-10-30T20:12:05Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5/presign.txt b/evidence/lab5/presign.txt new file mode 100644 index 00000000..740e9d5d --- /dev/null +++ b/evidence/lab5/presign.txt @@ -0,0 +1,11 @@ +# presign 2026-10-30T20:11:41Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/presign_flow.py --base-url "http://localhost:8080" +happy path: direct PUT 200, complete queued, ready after worker +message with attachment downloaded as the second chat member +repeated complete is idempotent +foreign complete: 404 +complete before PUT: 409 +wrong completion checksum: 409 +oversized direct PUT: 403 from S3; complete: 409 (nothing stored) +same-size overwrite: last PUT wins, checksum verified +PASS: lab4 presigned upload flow diff --git a/evidence/lab5/smoke.txt b/evidence/lab5/smoke.txt new file mode 100644 index 00000000..218f8d0e --- /dev/null +++ b/evidence/lab5/smoke.txt @@ -0,0 +1,3 @@ +# make test 2026-10-31T11:26:59Z +env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5/stand.txt b/evidence/lab5/stand.txt new file mode 100644 index 00000000..f6eb7495 --- /dev/null +++ b/evidence/lab5/stand.txt @@ -0,0 +1,20 @@ +# stand 2026-10-30T20:16:07Z +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp +mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp +mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 8 minutes ago Up 8 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp +mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 8 minutes ago Up 3 minutes 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp +mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 8 minutes ago Up 3 minutes (healthy) 5432/tcp +mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 8 minutes ago Up 4 minutes (healthy) 6379/tcp +mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8082->8082/tcp +mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 8 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp +mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8081->8081/tcp + +mtusi-messenger:lab5 10b83ff52c7d +rabbitmq:4.3.7-management-alpine 1528a88eebf2 +haproxy:3.2-alpine 254e6370f01c +redis:7.4-alpine 858f009f9709 +postgres:18-alpine 77f585114c32 +chrislusf/seaweedfs:3.80 1055999e08ee + +CPUs=2 MemTotal=2053644288 diff --git a/evidence/lab5/track2/run.txt b/evidence/lab5/track2/run.txt new file mode 100644 index 00000000..714163ca --- /dev/null +++ b/evidence/lab5/track2/run.txt @@ -0,0 +1,7 @@ +Dduplicate_before_effect rep=1: PASS +Dstale_version_after_v2 rep=1: PASS +Dduplicate_before_effect rep=2: PASS +Dstale_version_after_v2 rep=2: PASS +Dduplicate_before_effect rep=3: PASS +Dstale_version_after_v2 rep=3: PASS +PASS: track2 6/6 scenarios diff --git a/evidence/lab5/track2/runs.csv b/evidence/lab5/track2/runs.csv new file mode 100644 index 00000000..64ae4b21 --- /dev/null +++ b/evidence/lab5/track2/runs.csv @@ -0,0 +1,7 @@ +scenario,rep,pass,detail +duplicate_before_effect,1,1,attempts=1 +stale_version_after_v2,1,1,v1_job=01a18e6e-17b0-763b-b330-7adbcb43a700 +duplicate_before_effect,2,1,attempts=1 +stale_version_after_v2,2,1,v1_job=01a18e6e-455e-75f3-857a-750ed660378b +duplicate_before_effect,3,1,attempts=1 +stale_version_after_v2,3,1,v1_job=01a18e6e-74f2-7b4a-842b-c014c626413c diff --git a/evidence/lab5/track2/summary.md b/evidence/lab5/track2/summary.md new file mode 100644 index 00000000..f1e50c94 --- /dev/null +++ b/evidence/lab5/track2/summary.md @@ -0,0 +1,21 @@ +# Track 2 — versioned pipeline and duplicate/stale delivery + +Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`, +deterministic object keys, a serving version that changes only when all v2 +variants are committed, and stale-version acknowledgements, an at-least-once +pipeline loses no accepted work and never overwrites a newer result with an +older duplicate. + +Method: N=3 repetitions. Each repetition creates fresh users/chat and runs: +- duplicate pipeline publication while the worker is stopped, followed by recovery; +- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready. + +Invariants checked: unique variant rows per effect key; ready remains ready; +serving pipeline version never regresses; original bytes/checksum unchanged; +stale job state is not rewritten. + +Limitations: this harness covers duplicate and stale/reorder delivery. Crash +windows (before/after side effect, before/after ack, outbox gap) and lease +takeover are covered by student-tests/lab5_faults.py; both are single-node +Compose evidence, not a production availability claim. Delivery is +at-least-once; exactly-once is not claimed. diff --git a/evidence/lab5/unit-test.txt b/evidence/lab5/unit-test.txt new file mode 100644 index 00000000..1441fa49 --- /dev/null +++ b/evidence/lab5/unit-test.txt @@ -0,0 +1,41 @@ +env -u APP_ENV -u SERVER_HOST -u SERVER_PORT -u SERVER_READ_HEADER_TIMEOUT -u SERVER_READ_TIMEOUT -u SERVER_SHUTDOWN_TIMEOUT -u SERVER_MAX_BODY_BYTES -u LOG_LEVEL -u LOG_FORMAT -u POSTGRES_HOST -u POSTGRES_PORT -u POSTGRES_DATABASE -u POSTGRES_USER -u POSTGRES_PASSWORD -u POSTGRES_SSL_MODE -u POSTGRES_MAX_CONNS -u POSTGRES_MIN_CONNS -u POSTGRES_MAX_CONN_LIFETIME -u POSTGRES_MAX_CONN_IDLE_TIME -u POSTGRES_CONNECT_TIMEOUT -u POSTGRES_QUERY_TIMEOUT -u REDIS_ENABLED -u REDIS_HOST -u REDIS_PORT -u REDIS_PASSWORD -u REDIS_DB -u REDIS_MAX_RETRIES -u REDIS_DIAL_TIMEOUT -u REDIS_READ_TIMEOUT -u REDIS_WRITE_TIMEOUT -u REDIS_POOL_SIZE -u CACHE_ENABLED -u CACHE_CHAT_LIST_TTL -u CACHE_LOCK_TTL -u CACHE_LOCK_WAIT_TIMEOUT -u SESSION_TTL_SECONDS -u IDEMPOTENCY_TTL_SECONDS -u API_REPLICAS -u API_CPU_LIMIT -u API_MEM_LIMIT -u POSTGRES_MAX_CONNS -u POSTGRES_CPU_LIMIT -u POSTGRES_MEM_LIMIT -u LB_STATS_PORT -u S3_ENDPOINT -u S3_PUBLIC_ENDPOINT -u S3_PORT -u S3_REGION -u S3_BUCKET -u S3_ACCESS_KEY_ID -u S3_SECRET_ACCESS_KEY -u MAX_UPLOAD_BYTES -u MAX_IMAGE_PIXELS -u S3_PRESIGN_TTL -u S3_SWEEP_INTERVAL -u S3_SWEEP_GRACE -u BROKER_USER -u BROKER_PASSWORD -u BROKER_VHOST -u BROKER_URL -u BROKER_EXCHANGE -u BROKER_QUEUE -u BROKER_DLX -u BROKER_DLQ -u BROKER_MANAGEMENT_PORT -u BROKER_CPU_LIMIT -u BROKER_MEM_LIMIT -u WORKER_CONCURRENCY -u WORKER_PREFETCH -u WORKER_CPU_LIMIT -u WORKER_MEM_LIMIT -u WORKER_SHUTDOWN_TIMEOUT -u PROCESSING_PIPELINE_VERSION -u PROCESSING_MAX_ATTEMPTS -u PROCESSING_JOB_TIMEOUT -u PROCESSING_LEASE_TTL -u PROCESSING_HEARTBEAT -u PROCESSING_RETRY_BASE -u PROCESSING_RETRY_CAP -u OUTBOX_POLL_INTERVAL -u OUTBOX_CLAIM_TTL -u OUTBOX_BATCH -u OUTBOX_REAPER_INTERVAL -u OUTBOX_RETENTION -u RELAY_CPU_LIMIT -u RELAY_MEM_LIMIT -u STAGING_ORPHAN_GRACE -u STAGING_FAILED_RETENTION -u ADMISSION_MAX_PENDING_PER_USER -u ADMISSION_MAX_PENDING -u ADMISSION_MAX_STAGED_BYTES go test -race -shuffle=on -count=1 -timeout=5m ./... +? mtusi-messenger/cmd/jobctl [no test files] +? mtusi-messenger/cmd/messenger [no test files] +? mtusi-messenger/cmd/migrate [no test files] +? mtusi-messenger/cmd/relay [no test files] +? mtusi-messenger/cmd/s3init [no test files] +? mtusi-messenger/cmd/worker [no test files] +? mtusi-messenger/internal/broker [no test files] +ok mtusi-messenger/internal/config 1.717s +ok mtusi-messenger/internal/domain 2.647s +? mtusi-messenger/internal/failpoint [no test files] +ok mtusi-messenger/internal/health 1.315s +ok mtusi-messenger/internal/imagecheck 3.766s +ok mtusi-messenger/internal/imageproc 2.400s +ok mtusi-messenger/internal/logger 2.338s +? mtusi-messenger/internal/outbox [no test files] +ok mtusi-messenger/internal/platform/cache 4.909s +? mtusi-messenger/internal/platform/cache/mocks [no test files] +ok mtusi-messenger/internal/platform/objectstore 6.507s +ok mtusi-messenger/internal/platform/password 3.349s +ok mtusi-messenger/internal/platform/token 3.304s +ok mtusi-messenger/internal/service/attachment 5.105s +? mtusi-messenger/internal/service/attachment/mocks [no test files] +ok mtusi-messenger/internal/service/chat 4.460s +? mtusi-messenger/internal/service/chat/mocks [no test files] +ok mtusi-messenger/internal/service/message 4.119s +? mtusi-messenger/internal/service/message/mocks [no test files] +ok mtusi-messenger/internal/service/session 5.771s +? mtusi-messenger/internal/service/session/mocks [no test files] +ok mtusi-messenger/internal/service/user 6.251s +? mtusi-messenger/internal/service/user/mocks [no test files] +ok mtusi-messenger/internal/storage/postgres 6.484s +? mtusi-messenger/internal/storage/postgres/internal/sqlcgen [no test files] +? mtusi-messenger/internal/storage/postgres/migrations [no test files] +? mtusi-messenger/internal/storage/redis [no test files] +? mtusi-messenger/internal/storage/redis/mocks [no test files] +ok mtusi-messenger/internal/transport/rest 6.893s +? mtusi-messenger/internal/transport/rest/internal/oas [no test files] +? mtusi-messenger/internal/transport/rest/mocks [no test files] +? mtusi-messenger/internal/worker [no test files] +? mtusi-messenger/tools/specgen [no test files] diff --git a/evidence/lab5/validate.txt b/evidence/lab5/validate.txt new file mode 100644 index 00000000..5eafe4a9 --- /dev/null +++ b/evidence/lab5/validate.txt @@ -0,0 +1,3 @@ +# make validate PYTHON=.venv/bin/python 2026-10-30T22:22:13Z +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/lab5/variants.txt b/evidence/lab5/variants.txt new file mode 100644 index 00000000..224631cf --- /dev/null +++ b/evidence/lab5/variants.txt @@ -0,0 +1,3 @@ +# lab5_variants 2026-10-30T20:08:36Z +variants: thumbnail 256x160, crop 128x128, optimized WebP 320x200; metadata == bytes +PASS: lab5 variant contract diff --git a/evidence/lab5/vet.txt b/evidence/lab5/vet.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/lb-readiness.txt b/evidence/lb-readiness.txt index 00056343..89724218 100644 --- a/evidence/lb-readiness.txt +++ b/evidence/lb-readiness.txt @@ -3,23 +3,23 @@ # lb: haproxy:3.2-alpine ## 1. baseline: requests through LB by replica - 18 52861386fddb - 22 8587b4251b72 - stats: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + 18 52861386fddb + 22 8587b4251b72 + stats: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP ## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias instance-id: 9e9e905440d3, readiness while its database is reachable: 200 - load balancer slot for the candidate: api3, stats: api1=UP api2=UP api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + load balancer slot for the candidate: api3, stats: api1=UP api2=UP api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP ## 3. cut the candidate's database access (stop only the TCP proxy) candidate is still running: running, readiness: 503 waiting for the load balancer to mark api3 DOWN api3 went DOWN 2s after its database was cut (check: inter 2s fall 2) - stats after failure: api1=UP api2=UP api3=DOWN api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + stats after failure: api1=UP api2=UP api3=DOWN api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP ## 4. 40 requests through LB while the candidate is not ready - 18 52861386fddb - 22 8587b4251b72 + 18 52861386fddb + 22 8587b4251b72 (the candidate must not appear above) ## 5. business writes through LB during the exclusion @@ -28,4 +28,4 @@ "successful_rps": 1163.529, ## 6. cleanup - stats after cleanup: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP + stats after cleanup: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP diff --git a/go.mod b/go.mod index eba58d2f..5c21620c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.20.9 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4 github.com/aws/smithy-go v1.28.6 + github.com/gen2brain/webp v0.6.4 github.com/go-faster/errors v0.8.0 github.com/go-faster/jx v1.2.0 github.com/go-playground/validator/v10 v10.30.5 @@ -14,13 +15,15 @@ require ( github.com/jackc/pgx/v5 v5.11.0 github.com/ogen-go/ogen v1.24.0 github.com/pressly/goose/v3 v3.28.0 + github.com/rabbitmq/amqp091-go v1.16.0 github.com/redis/go-redis/v9 v9.22.0 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.12.1 go.uber.org/zap v1.28.0 go.uber.org/zap/exp v0.3.0 golang.org/x/crypto v0.57.0 - golang.org/x/sync v0.23.0 + golang.org/x/image v0.47.0 + golang.org/x/sync v0.24.0 ) require ( @@ -42,6 +45,7 @@ require ( github.com/cubicdaiya/gonp v1.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2 v1.12.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect github.com/fatih/color v1.19.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fatih/structtag v1.2.0 // indirect @@ -111,11 +115,11 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect golang.org/x/mod v0.41.0 // indirect - golang.org/x/net v0.58.0 // indirect - golang.org/x/sys v0.48.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sys v0.49.0 // indirect golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect - golang.org/x/tools v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + golang.org/x/tools v0.51.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a // indirect google.golang.org/grpc v1.83.2 // indirect diff --git a/go.sum b/go.sum index 2f490f36..051cef29 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -66,6 +68,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= +github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU= +github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-faster/errors v0.8.0 h1:9T9eJrM+72dFk7n4DfhuaDDe6cyuFCSW2oNUkN77Yqc= @@ -181,6 +185,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.28.0 h1:D2M+iL31GmpZxSHOhX8mqyqAT3CXnokUmm0eKoSP+Vc= github.com/pressly/goose/v3 v3.28.0/go.mod h1:v26MOuB8bL3kzzrt3Vqhb3R0PRVsl8hFQKdrht/L6Rk= +github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y= +github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -284,32 +290,34 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= +golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= diff --git a/internal/broker/rabbit.go b/internal/broker/rabbit.go new file mode 100644 index 00000000..934af619 --- /dev/null +++ b/internal/broker/rabbit.go @@ -0,0 +1,320 @@ +// Package broker owns the RabbitMQ topology, confirmed publishing and the +// consumer loop. Business state never lives here: PostgreSQL is the source of +// truth for jobs and the outbox. +package broker + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + amqp "github.com/rabbitmq/amqp091-go" + + "mtusi-messenger/internal/domain" +) + +type Config struct { + URL string + Exchange string + DLX string + Queue string + DLQ string + ConnectTimeout time.Duration + ConfirmTimeout time.Duration + Heartbeat time.Duration + Prefetch int +} + +type Client struct { + cfg Config + + conn *amqp.Connection + publisher *amqp.Channel + + mu sync.Mutex + returns chan amqp.Return + closed chan *amqp.Error +} + +// Dial connects, declares the durable topology idempotently and opens a +// confirm-mode publisher channel. +func Dial(cfg Config) (*Client, error) { + if cfg.URL == "" { + return nil, errors.New("broker url is empty") + } + + conn, err := amqp.DialConfig(cfg.URL, amqp.Config{ + Heartbeat: cfg.Heartbeat, + Dial: amqp.DefaultDial(cfg.ConnectTimeout), + Properties: amqp.Table{ + "connection_name": "messenger-relay", + }, + }) + if err != nil { + return nil, fmt.Errorf("dial broker: %w", err) + } + + channel, err := conn.Channel() + if err != nil { + _ = conn.Close() + + return nil, fmt.Errorf("open topology channel: %w", err) + } + + if err := declareTopology(channel, cfg); err != nil { + _ = channel.Close() + _ = conn.Close() + + return nil, err + } + + if err := channel.Close(); err != nil { + _ = conn.Close() + + return nil, fmt.Errorf("close topology channel: %w", err) + } + + publisher, err := conn.Channel() + if err != nil { + _ = conn.Close() + + return nil, fmt.Errorf("open publisher channel: %w", err) + } + + client := &Client{ + cfg: cfg, + conn: conn, + publisher: publisher, + returns: make(chan amqp.Return, 16), + closed: conn.NotifyClose(make(chan *amqp.Error, 1)), + } + + if err := publisher.Confirm(false); err != nil { + _ = client.Close() + + return nil, fmt.Errorf("enable publisher confirms: %w", err) + } + + publisher.NotifyReturn(client.returns) + + return client, nil +} + +func declareTopology(channel *amqp.Channel, cfg Config) error { + if err := channel.ExchangeDeclare(cfg.Exchange, "direct", true, false, false, false, nil); err != nil { + return fmt.Errorf("declare exchange: %w", err) + } + + if err := channel.ExchangeDeclare(cfg.DLX, "direct", true, false, false, false, nil); err != nil { + return fmt.Errorf("declare dlx: %w", err) + } + + queue, err := channel.QueueDeclare(cfg.Queue, true, false, false, false, amqp.Table{ + "x-queue-type": "quorum", + "x-delivery-limit": int32(100), + "x-delayed-retry-type": "all", + "x-delayed-retry-min": int32(5000), + "x-delayed-retry-max": int32(60000), + "x-dead-letter-exchange": cfg.DLX, + "x-dead-letter-routing-key": cfg.DLQ, + "x-dead-letter-strategy": "at-least-once", + "x-overflow": "reject-publish", + "x-max-length": int32(100000), + "x-consumer-timeout": int32(300000), + }) + if err != nil { + return fmt.Errorf("declare job queue: %w", err) + } + + if queue.Name != cfg.Queue { + return fmt.Errorf("broker returned queue %q, want %q", queue.Name, cfg.Queue) + } + + if err := channel.QueueBind(cfg.Queue, cfg.Queue, cfg.Exchange, false, nil); err != nil { + return fmt.Errorf("bind job queue: %w", err) + } + + if _, err := channel.QueueDeclare(cfg.DLQ, true, false, false, false, amqp.Table{ + "x-queue-type": "quorum", + "x-max-length": int32(100000), + "x-overflow": "reject-publish", + "x-message-ttl": int32(7 * 24 * 60 * 60 * 1000), + }); err != nil { + return fmt.Errorf("declare dlq: %w", err) + } + + if err := channel.QueueBind(cfg.DLQ, cfg.DLQ, cfg.DLX, false, nil); err != nil { + return fmt.Errorf("bind dlq: %w", err) + } + + return nil +} + +func (c *Client) Connected() bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.conn != nil && !c.conn.IsClosed() +} + +// PublishConfirmed returns only after the broker acknowledged the persistent +// message. A duplicate publication after a crash is normal and harmless. +func (c *Client) PublishConfirmed(ctx context.Context, routingKey string, message domain.JobMessage) error { + payload, err := message.MarshalPayload() + if err != nil { + return fmt.Errorf("marshal job message: %w", err) + } + + confirmation, err := c.publisher.PublishWithDeferredConfirmWithContext( + ctx, + c.cfg.Exchange, + routingKey, + true, + false, + amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: message.JobID.String(), + Type: message.Type, + Timestamp: message.CreatedAt, + Body: payload, + }, + ) + if err != nil { + return fmt.Errorf("publish job: %w", err) + } + + confirmCtx := ctx + if c.cfg.ConfirmTimeout > 0 { + var cancel context.CancelFunc + confirmCtx, cancel = context.WithTimeout(ctx, c.cfg.ConfirmTimeout) + defer cancel() + } + + confirmed, err := confirmation.WaitContext(confirmCtx) + if err != nil { + return fmt.Errorf("await publisher confirm: %w", err) + } + + if !confirmed { + return errors.New("broker nacked the job message") + } + + // mandatory=true: an unroutable message is returned even though the + // confirm may still be positive. The topology is declared before publish, + // so this is a safety net rather than a normal path. + select { + case returned := <-c.returns: + if returned.MessageId == message.JobID.String() { + return fmt.Errorf("broker returned unroutable message: %s", returned.ReplyText) + } + default: + } + + return nil +} + +type Outcome int + +const ( + Ack Outcome = iota + Requeue + Drop +) + +type Handler func(ctx context.Context, delivery amqp.Delivery) Outcome + +// Consume blocks until the context is cancelled or the consumer channel fails. +// Each call uses its own channel so worker slots do not share prefetch state. +func (c *Client) Consume(ctx context.Context, handler Handler) error { + channel, err := c.conn.Channel() + if err != nil { + return fmt.Errorf("open consumer channel: %w", err) + } + defer channel.Close() + + if err := channel.Qos(c.cfg.Prefetch, 0, false); err != nil { + return fmt.Errorf("set qos: %w", err) + } + + consumerTag := fmt.Sprintf("messenger-%d", time.Now().UnixNano()) + + deliveries, err := channel.Consume( + c.cfg.Queue, + consumerTag, + false, + false, + false, + false, + nil, + ) + if err != nil { + return fmt.Errorf("consume: %w", err) + } + + done := make(chan struct{}) + defer close(done) + + go func() { + select { + case <-ctx.Done(): + _ = channel.Cancel(consumerTag, false) + case <-done: + } + }() + + for delivery := range deliveries { + outcome := safeHandle(ctx, handler, delivery) + + switch outcome { + case Ack: + _ = delivery.Ack(false) + case Requeue: + _ = delivery.Nack(false, true) + case Drop: + _ = delivery.Nack(false, false) + } + + if ctx.Err() != nil { + // Let unacked deliveries return to the broker on channel close. + return nil + } + } + + if ctx.Err() != nil { + return nil + } + + return errors.New("consumer channel closed") +} + +func safeHandle(ctx context.Context, handler Handler, delivery amqp.Delivery) (outcome Outcome) { + defer func() { + if recover() != nil { + outcome = Requeue + } + }() + + return handler(ctx, delivery) +} + +func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.publisher != nil { + _ = c.publisher.Close() + c.publisher = nil + } + + if c.conn != nil { + err := c.conn.Close() + c.conn = nil + + return err + } + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index addb8c10..a1c7a0d0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,6 +37,75 @@ type Config struct { Session Session `mapstructure:"session"` Idempotency Idempotency `mapstructure:"idempotency"` S3 S3 `mapstructure:"s3"` + Broker Broker `mapstructure:"broker"` + Worker Worker `mapstructure:"worker"` + Relay Relay `mapstructure:"relay"` + Outbox Outbox `mapstructure:"outbox"` + Processing Processing `mapstructure:"processing"` + Staging Staging `mapstructure:"staging"` + Admission Admission `mapstructure:"admission"` +} + +// Broker carries the AMQP transport settings. The API never publishes; relay +// and worker require a non-empty URL at startup. +type Broker struct { + URL string `mapstructure:"url"` + ConnectTimeout time.Duration `mapstructure:"connect_timeout"` + ConfirmTimeout time.Duration `mapstructure:"confirm_timeout"` + Heartbeat time.Duration `mapstructure:"heartbeat"` + Exchange string `mapstructure:"exchange" validate:"required"` + DLX string `mapstructure:"dlx" validate:"required"` + Queue string `mapstructure:"queue" validate:"required"` + DLQ string `mapstructure:"dlq" validate:"required"` + Prefetch int `mapstructure:"prefetch" validate:"required,gt=0"` +} + +// Worker bounds the process that consumes jobs. +type Worker struct { + Concurrency int `mapstructure:"concurrency" validate:"required,gt=0,lte=32"` + Prefetch int `mapstructure:"prefetch" validate:"required,gt=0,lte=100"` + ShutdownTimeout time.Duration `mapstructure:"shutdown_timeout" validate:"required,gt=0"` + HealthAddr string `mapstructure:"health_addr" validate:"required"` +} + +// Relay bounds the outbox publisher and lease reaper. +type Relay struct { + HealthAddr string `mapstructure:"health_addr" validate:"required"` +} + +// Outbox schedules publications and reclaims abandoned relay claims. +type Outbox struct { + PollInterval time.Duration `mapstructure:"poll_interval" validate:"required,gt=0"` + ClaimTTL time.Duration `mapstructure:"claim_ttl" validate:"required,gt=0"` + Batch int `mapstructure:"batch" validate:"required,gt=0,lte=1000"` + ReaperInterval time.Duration `mapstructure:"reaper_interval" validate:"required,gt=0"` + ReconcileAfter time.Duration `mapstructure:"reconcile_after" validate:"omitempty,gt=0"` + Retention time.Duration `mapstructure:"retention" validate:"required,gt=0"` +} + +// Processing holds the shared job contract knobs read by API and worker. +type Processing struct { + PipelineVersion int `mapstructure:"pipeline_version" validate:"required,gte=1,lte=2"` + MaxAttempts int `mapstructure:"max_attempts" validate:"required,gte=1,lte=100"` + JobTimeout time.Duration `mapstructure:"job_timeout" validate:"required,gt=0"` + LeaseTTL time.Duration `mapstructure:"lease_ttl" validate:"required,gt=0"` + Heartbeat time.Duration `mapstructure:"heartbeat" validate:"required,gt=0"` + RetryBase time.Duration `mapstructure:"retry_base" validate:"required,gt=0"` + RetryCap time.Duration `mapstructure:"retry_cap" validate:"required,gt=0"` +} + +// Staging controls cleanup of objects that are no longer the active source. +type Staging struct { + OrphanGrace time.Duration `mapstructure:"orphan_grace" validate:"required,gt=0"` + FailedRetention time.Duration `mapstructure:"failed_retention" validate:"required,gt=0"` +} + +// Admission bounds accepted-but-unfinished work per user and globally. +type Admission struct { + MaxPendingPerUser int `mapstructure:"max_pending_per_user" validate:"required,gte=0"` + MaxPending int `mapstructure:"max_pending" validate:"required,gte=0"` + MaxStagedBytes int64 `mapstructure:"max_staged_bytes" validate:"required,gte=0"` + MaxConcurrent int `mapstructure:"max_concurrent_uploads" validate:"omitempty,gt=0"` } // S3 describes the object store and attachment limits. Access keys are checked @@ -178,6 +247,38 @@ var defaults = map[string]any{ "cache.chat_list_ttl": "5m", "cache.lock_ttl": "10s", "cache.lock_wait_timeout": "2s", + + "broker.url": "", + "broker.connect_timeout": "5s", + "broker.confirm_timeout": "5s", + "broker.heartbeat": "10s", + "broker.exchange": "attachments.jobs", + "broker.dlx": "attachments.dlx", + "broker.queue": "attachment.process.v1", + "broker.dlq": "attachment.process.dead", + "broker.prefetch": 1, + "worker.concurrency": 1, + "worker.prefetch": 1, + "worker.shutdown_timeout": "30s", + "worker.health_addr": ":8081", + "relay.health_addr": ":8082", + "outbox.poll_interval": "1s", + "outbox.claim_ttl": "30s", + "outbox.batch": 20, + "outbox.reaper_interval": "15s", + "outbox.retention": "168h", + "processing.pipeline_version": 1, + "processing.max_attempts": 5, + "processing.job_timeout": "120s", + "processing.lease_ttl": "180s", + "processing.heartbeat": "30s", + "processing.retry_base": "2s", + "processing.retry_cap": "60s", + "staging.orphan_grace": "1h", + "staging.failed_retention": "168h", + "admission.max_pending_per_user": 20, + "admission.max_pending": 200, + "admission.max_staged_bytes": 2147483648, } func Load() (Config, error) { @@ -233,6 +334,28 @@ func (c Config) validateDependencies() error { return errors.New("validate config: s3.sweep_grace must exceed s3.presign_ttl") } + if c.Processing.LeaseTTL < 3*c.Processing.Heartbeat { + return errors.New("validate config: processing.lease_ttl must be at least 3x processing.heartbeat") + } + + if c.Processing.JobTimeout >= c.Processing.LeaseTTL { + return errors.New("validate config: processing.job_timeout must be below processing.lease_ttl") + } + + if c.Processing.RetryCap < c.Processing.RetryBase { + return errors.New("validate config: processing.retry_cap must be >= processing.retry_base") + } + + if c.Staging.FailedRetention < c.Staging.OrphanGrace { + return errors.New("validate config: staging.failed_retention must be >= staging.orphan_grace") + } + + if c.Broker.URL != "" { + if _, err := url.Parse(c.Broker.URL); err != nil { + return fmt.Errorf("validate config: broker.url: %w", err) + } + } + return nil } diff --git a/internal/domain/attachment.go b/internal/domain/attachment.go index 9239141c..fabf698c 100644 --- a/internal/domain/attachment.go +++ b/internal/domain/attachment.go @@ -6,53 +6,127 @@ import ( "github.com/google/uuid" ) -// AttachmentStatus is the lifecycle of an attachment. Only ready attachments -// are exposed to members; pending rows exist while the object is being written, -// failed rows are reserved for later labs. +// AttachmentStatus is the public lifecycle of an attachment. Pending is an +// internal presign state and is never exposed by the API. type AttachmentStatus string const ( - AttachmentPending AttachmentStatus = "pending" - AttachmentReady AttachmentStatus = "ready" - AttachmentFailed AttachmentStatus = "failed" + AttachmentPending AttachmentStatus = "pending" + AttachmentQueued AttachmentStatus = "queued" + AttachmentProcessing AttachmentStatus = "processing" + AttachmentReady AttachmentStatus = "ready" + AttachmentFailed AttachmentStatus = "failed" ) func (s AttachmentStatus) Valid() bool { switch s { - case AttachmentPending, AttachmentReady, AttachmentFailed: + case AttachmentPending, AttachmentQueued, AttachmentProcessing, AttachmentReady, AttachmentFailed: return true default: return false } } -// Attachment describes the original object. ObjectKey is internal: it never -// leaves the service and must not be mapped into an API response. -type Attachment struct { - ID uuid.UUID - ChatID uuid.UUID - OwnerID uuid.UUID - Filename string +// Public reports whether the status is part of the API enum. +func (s AttachmentStatus) Public() bool { + switch s { + case AttachmentQueued, AttachmentProcessing, AttachmentReady, AttachmentFailed: + return true + default: + return false + } +} + +// VariantName is the contract variant identifier. +type VariantName string + +const ( + VariantOriginal VariantName = "original" + VariantThumbnail VariantName = "thumbnail" + VariantCrop VariantName = "crop" + VariantOptimized VariantName = "optimized" +) + +func (n VariantName) Valid() bool { + switch n { + case VariantThumbnail, VariantCrop, VariantOptimized: + return true + default: + return false + } +} + +// ContentType returns the contract media type of the variant. +func (n VariantName) ContentType() string { + if n == VariantOptimized { + return "image/webp" + } + + return "image/jpeg" +} + +// ObjectName returns the object suffix for the variant. +func (n VariantName) ObjectName() string { + if n == VariantOptimized { + return string(n) + ".webp" + } + + return string(n) + ".jpg" +} + +// Variant is one derived image. ObjectKey is internal and never leaves the +// service. +type Variant struct { + Name VariantName ContentType string SizeBytes int64 + Width int + Height int SHA256 string - Status AttachmentStatus - ErrorCode *string ObjectKey string - CreatedAt time.Time - UpdatedAt time.Time +} + +// Attachment describes the original object and the current pipeline result. +type Attachment struct { + ID uuid.UUID + ChatID uuid.UUID + OwnerID uuid.UUID + Filename string + ContentType string + SizeBytes int64 + SHA256 string + Status AttachmentStatus + ErrorCode *string + ObjectKey string + StagingKey string + PipelineVersion int + Variants []Variant + CreatedAt time.Time + UpdatedAt time.Time + ReadyAt *time.Time + FailedAt *time.Time +} + +// AdmissionLimits bounds accepted-but-unfinished work. Zero means unlimited. +type AdmissionLimits struct { + PerUser int + Total int + Bytes int64 } // AttachmentDraft is a fully validated attachment awaiting persistence. type AttachmentDraft struct { - ID uuid.UUID - ChatID uuid.UUID - OwnerID uuid.UUID - ObjectKey string - Filename string - ContentType string - SizeBytes int64 - SHA256 string + ID uuid.UUID + ChatID uuid.UUID + OwnerID uuid.UUID + ObjectKey string + StagingKey string + Filename string + ContentType string + SizeBytes int64 + SHA256 string + Status AttachmentStatus + PipelineVersion int } // AttachmentState is the minimal projection used while validating message @@ -70,6 +144,13 @@ type AttachmentKey struct { CreatedAt time.Time } +// StaleStaging is one staging object whose owner row is already terminal. +type StaleStaging struct { + AttachmentID uuid.UUID + StagingKey string + Status AttachmentStatus +} + // PresignRequest is the validated client intent for a direct upload. type PresignRequest struct { ChatID uuid.UUID diff --git a/internal/domain/errors.go b/internal/domain/errors.go index 9844b441..c5823285 100644 --- a/internal/domain/errors.go +++ b/internal/domain/errors.go @@ -15,4 +15,5 @@ var ( ErrTooLarge = errors.New("payload is too large") ErrUnsupportedMedia = errors.New("unsupported media type") ErrIncomplete = errors.New("object is not fully uploaded") + ErrRateLimited = errors.New("too many unfinished uploads") ) diff --git a/internal/domain/job.go b/internal/domain/job.go new file mode 100644 index 00000000..9ec6c65b --- /dev/null +++ b/internal/domain/job.go @@ -0,0 +1,180 @@ +package domain + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" +) + +// JobType is the fixed contract type from contracts/jobs.schema.json. +const JobType = "attachment.process.v1" + +// JobStatus is the durable state of one logical processing job. Superseded is +// internal: a newer pipeline version replaced the job before it completed. +type JobStatus string + +const ( + JobQueued JobStatus = "queued" + JobProcessing JobStatus = "processing" + JobReady JobStatus = "ready" + JobFailed JobStatus = "failed" + JobSuperseded JobStatus = "superseded" +) + +func (s JobStatus) Valid() bool { + switch s { + case JobQueued, JobProcessing, JobReady, JobFailed, JobSuperseded: + return true + default: + return false + } +} + +// Job is the durable processing intent. SourceKey is the trusted object the +// worker reads: staging for a fresh upload, the original object for reprocess. +type Job struct { + ID uuid.UUID + AttachmentID uuid.UUID + PipelineVersion int + SourceKey string + Attempt int + MaxAttempts int + Status JobStatus + AvailableAt time.Time + LeaseOwner string + LeaseExpiresAt *time.Time + LastErrorCode *string + ReplayCount int + CreatedAt time.Time + UpdatedAt time.Time + CompletedAt *time.Time +} + +// OutboxMessage is one pending broker publication. Payload is the exact +// jobs.schema.json document; no binary data or credentials are stored. +type OutboxMessage struct { + ID int64 + JobID uuid.UUID + Attempt int + Kind string + RoutingKey string + Payload []byte + AvailableAt time.Time + PublishedAt *time.Time + ClaimedBy string + ClaimedUntil *time.Time + PublishAttempts int + LastError *string + CreatedAt time.Time +} + +// JobMessage is the broker carrier defined by contracts/jobs.schema.json. +type JobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +// NewJobMessage builds the only message the application is allowed to publish. +func NewJobMessage(job Job) JobMessage { + return JobMessage{ + SchemaVersion: 1, + JobID: job.ID, + Type: JobType, + AttachmentID: job.AttachmentID, + PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, + CreatedAt: job.CreatedAt.UTC(), + } +} + +// Validate enforces the closed schema. The worker must never trust paths or +// storage metadata from this message. +func (m JobMessage) Validate() error { + if m.SchemaVersion != 1 { + return errors.New("unsupported schema_version") + } + + if m.Type != JobType { + return fmt.Errorf("unsupported type %q", m.Type) + } + + if m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil { + return errors.New("job_id and attachment_id are required") + } + + if m.PipelineVersion < 1 || m.Attempt < 1 { + return errors.New("pipeline_version and attempt must be >= 1") + } + + if m.CreatedAt.IsZero() { + return errors.New("created_at is required") + } + + return nil +} + +func (m JobMessage) MarshalPayload() ([]byte, error) { + return json.Marshal(m) +} + +func ParseJobMessage(payload []byte) (JobMessage, error) { + var message JobMessage + + if err := json.Unmarshal(payload, &message); err != nil { + return JobMessage{}, fmt.Errorf("decode job message: %w", err) + } + + if err := message.Validate(); err != nil { + return JobMessage{}, err + } + + return message, nil +} + +// ClaimOutcome explains why a delivery did or did not become the active +// processing attempt. The worker maps it to ack/nack without guessing. +type ClaimOutcome string + +const ( + ClaimAcquired ClaimOutcome = "acquired" + ClaimAlreadyReady ClaimOutcome = "already_ready" + ClaimAlreadyFailed ClaimOutcome = "already_failed" + ClaimSuperseded ClaimOutcome = "superseded" + ClaimStaleAttempt ClaimOutcome = "stale_attempt" + ClaimFutureAttempt ClaimOutcome = "future_attempt" + ClaimBusyLease ClaimOutcome = "busy_lease" + ClaimNotDue ClaimOutcome = "not_due" + ClaimVersionStale ClaimOutcome = "version_stale" + ClaimJobNotFound ClaimOutcome = "job_not_found" + ClaimSourceMissing ClaimOutcome = "source_missing" + ClaimPayloadInvalid ClaimOutcome = "payload_invalid" +) + +type ClaimResult struct { + Outcome ClaimOutcome + Job Job + Attachment Attachment +} + +// Job event names used by the operator audit trail. +const ( + EventEnqueued = "enqueued" + EventClaimed = "claimed" + EventRetried = "retried" + EventReady = "ready" + EventFailed = "failed" + EventLeaseExpired = "lease_expired" + EventReplayed = "replayed" + EventReprocessed = "reprocessed" + EventSuperseded = "superseded" +) diff --git a/internal/domain/keys.go b/internal/domain/keys.go new file mode 100644 index 00000000..d99ab7b0 --- /dev/null +++ b/internal/domain/keys.go @@ -0,0 +1,31 @@ +package domain + +import ( + "strconv" + + "github.com/google/uuid" +) + +const ( + StagingUploadPrefix = "staging/uploads/" + StagingVerifiedPrefix = "staging/verified/" + StagingPresignPrefix = "staging/presign/" + AttachmentsPrefix = "attachments/" +) + +func StagingUploadKey(chatID, attachmentID uuid.UUID) string { + return StagingUploadPrefix + chatID.String() + "/" + attachmentID.String() +} + +func StagingVerifiedKey(chatID, attachmentID uuid.UUID) string { + return StagingVerifiedPrefix + chatID.String() + "/" + attachmentID.String() +} + +func OriginalObjectKey(chatID, attachmentID uuid.UUID) string { + return AttachmentsPrefix + chatID.String() + "/" + attachmentID.String() + "/original" +} + +func VariantObjectKey(chatID, attachmentID uuid.UUID, pipelineVersion int, name VariantName) string { + return AttachmentsPrefix + chatID.String() + "/" + attachmentID.String() + + "/variants/v" + strconv.Itoa(pipelineVersion) + "/" + name.ObjectName() +} diff --git a/internal/failpoint/failpoint.go b/internal/failpoint/failpoint.go new file mode 100644 index 00000000..c43cf00c --- /dev/null +++ b/internal/failpoint/failpoint.go @@ -0,0 +1,107 @@ +// Package failpoint implements deterministic, environment-gated crash windows +// for the lab5 fault-injection evidence. Nothing fires unless FAULTS_ENABLED=1 +// and the action matches both FAULT_ and, for job-scoped faults, +// FAULT_TARGET_JOB. Production and CI leave the variables unset. +package failpoint + +import ( + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/google/uuid" +) + +// Enabled reports whether fault injection is explicitly switched on. +func Enabled() bool { + if os.Getenv("FAULTS_ENABLED") != "1" { + return false + } + + env := os.Getenv("APP_ENV") + return env == "" || env == "dev" +} + +// Check exits the process if the action is armed for this job. It must be +// called at the exact window the evidence describes; a marker line is written +// before exit so the harness can prove the fault actually fired. +func Check(action string, jobID uuid.UUID) { + if !Enabled() { + return + } + + if os.Getenv("FAULT_"+action) != "exit" { + return + } + + target := os.Getenv("FAULT_TARGET_JOB") + if target != "" && target != jobID.String() { + return + } + + // A one-shot marker survives the container restart performed by the + // restart policy. Without it a targeted crash failpoint would loop forever. + marker := filepath.Join(os.TempDir(), "messenger-failpoint-"+action+"-"+jobID.String()) + if _, err := os.Stat(marker); err == nil { + return + } + + _ = os.WriteFile(marker, []byte("fired\n"), 0o600) + + slog.Default().Error("failpoint fired", + slog.String("action", action), + slog.String("job_id", jobID.String()), + slog.Int("exit_code", 70), + ) + _ = os.Stdout.Sync() + _ = os.Stderr.Sync() + + os.Exit(70) +} + +// Mode returns the armed value for an action/job pair. Unlike Check it does not +// exit; the worker uses it to inject a permanent or transient processing error. +func Mode(action string, jobID uuid.UUID) string { + if !Enabled() { + return "" + } + + target := os.Getenv("FAULT_TARGET_JOB") + if target != "" && target != jobID.String() { + return "" + } + + return os.Getenv("FAULT_" + action) +} + +// Sleep blocks for the configured duration when the action is armed for this +// job. It is used for hang/slow-worker scenarios where the heartbeat must keep +// running (the caller controls that separately). +func Sleep(action string, jobID uuid.UUID) { + if !Enabled() { + return + } + + raw := os.Getenv("FAULT_" + action) + if raw == "" { + return + } + + target := os.Getenv("FAULT_TARGET_JOB") + if target != "" && target != jobID.String() { + return + } + + delay, err := time.ParseDuration(raw) + if err != nil { + return + } + + slog.Default().Warn("failpoint sleeping", + slog.String("action", action), + slog.String("job_id", jobID.String()), + slog.Duration("duration", delay), + ) + time.Sleep(delay) +} diff --git a/internal/imagecheck/imagecheck.go b/internal/imagecheck/imagecheck.go index 3349c904..c746f884 100644 --- a/internal/imagecheck/imagecheck.go +++ b/internal/imagecheck/imagecheck.go @@ -1,12 +1,16 @@ // Package imagecheck validates PNG and JPEG structure without decoding the -// raster. A full decode of a 40-megapixel image would allocate hundreds of -// megabytes; attachment uploads must stay bounded. The parser verifies chunk -// CRCs and the JPEG marker/scan structure, which catches truncation and -// corruption while using constant memory. +// raster into an image. A full decode of a 40-megapixel image would allocate +// hundreds of megabytes; attachment uploads must stay bounded. For PNG the +// parser verifies chunk CRCs and streams the zlib image data to check exact +// scanline sizes and filter types without keeping the raster in memory. The +// JPEG parser verifies marker/scan structure. Both catch truncation and +// corruption while using bounded memory. package imagecheck import ( "bufio" + "bytes" + "compress/zlib" "encoding/binary" "errors" "fmt" @@ -73,10 +77,20 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) { } var ( - info Info - seen = make(map[string]bool, 4) - first = true - hasIDAT bool + info Info + seen = make(map[string]bool, 4) + first = true + hasIDAT bool + paletteReady bool + trnsSeen bool + colorType byte + bitDepth byte + paletteEntries int + idat bytes.Buffer + raster []pngPass + rasterBytes int64 + maxIDAT int64 = -1 + lastChunk string ) info.ContentType = "image/png" @@ -107,11 +121,32 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) { first = false + if seen[chunkType] && (chunkType == "IHDR" || chunkType == "PLTE" || chunkType == "IEND") { + return Info{}, corruptf("png duplicate %s chunk", chunkType) + } + + if chunkType == "PLTE" && hasIDAT { + return Info{}, corruptf("png PLTE chunk after IDAT") + } + + if chunkType == "IDAT" && seen["IDAT"] && lastChunk != "IDAT" { + return Info{}, corruptf("png IDAT chunks must be consecutive") + } + if err := checkCriticalChunk(chunkType, seen); err != nil { return Info{}, err } - data, err := readChunkData(reader, checksum, length) + var chunkSink io.Writer + if chunkType == "IDAT" { + if maxIDAT >= 0 && int64(idat.Len())+length > maxIDAT { + return Info{}, corruptf("png idat compressed data exceeds %d bytes", maxIDAT) + } + + chunkSink = &idat + } + + data, err := readChunkData(reader, checksum, length, chunkSink) if err != nil { return Info{}, corrupt("png chunk data", err) } @@ -133,6 +168,8 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) { info.Width = int64(binary.BigEndian.Uint32(data[0:4])) info.Height = int64(binary.BigEndian.Uint32(data[4:8])) + colorType = data[9] + bitDepth = data[8] if info.Width <= 0 || info.Height <= 0 { return Info{}, corruptf("png dimensions %dx%d", info.Width, info.Height) @@ -143,8 +180,84 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) { if maxPixels > 0 && (info.Width > maxPixels || info.Height > maxPixels || info.Width*info.Height > maxPixels) { return Info{}, fmt.Errorf("%w: %dx%d", ErrTooLarge, info.Width, info.Height) } + + bound, err := pngRasterLayout(info, data[8], data[9], data[10], data[11], data[12]) + if err != nil { + return Info{}, err + } + + raster = bound + rasterBytes = pngRasterBytes(bound) + if rasterBytes <= 1<<40 { + // zlib overhead: ~5 bytes per 16 KiB block, 2-byte header and + // a 4-byte Adler trailer. The 1 MiB constant covers tiny + // images where the relative term vanishes. + maxIDAT = rasterBytes + rasterBytes/256 + 1<<20 + } case "IDAT": hasIDAT = true + case "PLTE": + if colorType == 0 || colorType == 4 { + return Info{}, corruptf("png PLTE is invalid for color type %d", colorType) + } + + if length == 0 || length%3 != 0 || length > 3*256 { + return Info{}, corruptf("png PLTE length %d", length) + } + + if length/3 > 1< 256 { + return Info{}, corruptf("png tRNS length %d", length) + } + + if int(length) > paletteEntries { + return Info{}, corruptf( + "png tRNS length %d exceeds %d palette entries", + length, paletteEntries, + ) + } + default: + return Info{}, corruptf("png tRNS invalid for color type %d", colorType) + } + + trnsSeen = true case "IEND": if length != 0 { return Info{}, corruptf("png IEND length %d", length) @@ -154,18 +267,200 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) { return Info{}, corruptf("png has no IDAT chunk") } + if colorType == 3 && !paletteReady { + return Info{}, corruptf("png palette image has no PLTE before IDAT") + } + + // The chunk walker proves structure, but a stream whose bytes + // merely look like a chunk layout (valid length/type/CRC) may + // still not be a PNG raster: the zlib payload must inflate to + // exactly the IHDR-declared scanlines with valid filter bytes. + if err := validateRaster(idat.Bytes(), raster); err != nil { + return Info{}, err + } + if _, err := io.Copy(io.Discard, reader); err != nil { return Info{}, corrupt("png trailing data", err) } return info, nil } + + lastChunk = chunkType + } +} + +// pngPass is one consecutive block of scanlines in the inflated image stream. +// Non-interlaced PNGs have a single pass; Adam7 has up to seven. +type pngPass struct { + rowBytes int64 + rows int64 +} + +type pngPassSpec struct { + x0, y0 int64 + dx, dy int64 +} + +var adam7Passes = []pngPassSpec{ + {x0: 0, y0: 0, dx: 8, dy: 8}, + {x0: 4, y0: 0, dx: 8, dy: 8}, + {x0: 0, y0: 4, dx: 4, dy: 8}, + {x0: 2, y0: 0, dx: 4, dy: 4}, + {x0: 0, y0: 2, dx: 2, dy: 4}, + {x0: 1, y0: 0, dx: 2, dy: 2}, + {x0: 0, y0: 1, dx: 1, dy: 2}, +} + +// pngRasterLayout returns the exact inflated size and row layout for the +// IHDR. The Adam7 layout matters: a single non-interlaced formula either +// rejects valid interlaced images or accepts oversized streams. +func pngRasterLayout( + info Info, + bitDepth, colorType, compression, filter, interlace byte, +) ([]pngPass, error) { + if compression != 0 || filter != 0 { + return nil, corruptf( + "png compression/filter method %d/%d", compression, filter, + ) + } + + var bitsPerPixel int64 + + switch colorType { + case 0: // grayscale + if bitDepth != 1 && bitDepth != 2 && bitDepth != 4 && bitDepth != 8 && bitDepth != 16 { + return nil, corruptf("png invalid bit depth %d for color type 0", bitDepth) + } + + bitsPerPixel = int64(bitDepth) + case 2: // truecolor + if bitDepth != 8 && bitDepth != 16 { + return nil, corruptf("png invalid bit depth %d for color type 2", bitDepth) + } + + bitsPerPixel = 3 * int64(bitDepth) + case 3: // palette + if bitDepth != 1 && bitDepth != 2 && bitDepth != 4 && bitDepth != 8 { + return nil, corruptf("png invalid bit depth %d for color type 3", bitDepth) + } + + bitsPerPixel = int64(bitDepth) + case 4: // grayscale + alpha + if bitDepth != 8 && bitDepth != 16 { + return nil, corruptf("png invalid bit depth %d for color type 4", bitDepth) + } + + bitsPerPixel = 2 * int64(bitDepth) + case 6: // RGBA + if bitDepth != 8 && bitDepth != 16 { + return nil, corruptf("png invalid bit depth %d for color type 6", bitDepth) + } + + bitsPerPixel = 4 * int64(bitDepth) + default: + return nil, corruptf("png invalid color type %d", colorType) + } + + specs := []pngPassSpec{{x0: 0, y0: 0, dx: 1, dy: 1}} + switch interlace { + case 0: + case 1: + specs = adam7Passes + default: + return nil, corruptf("png invalid interlace method %d", interlace) + } + + passes := make([]pngPass, 0, len(specs)) + + for _, spec := range specs { + if info.Width <= spec.x0 || info.Height <= spec.y0 { + continue + } + + width := (info.Width - spec.x0 + spec.dx - 1) / spec.dx + rows := (info.Height - spec.y0 + spec.dy - 1) / spec.dy + rowBytes := (width*bitsPerPixel + 7) / 8 + + // Keep the total overflow-safe. Valid PNGs are additionally bounded + // by the pixel limit checked by the caller. + if rowBytes+1 > (1<<62)/rows { + return nil, fmt.Errorf("%w: png dimensions overflow", ErrTooLarge) + } + + passes = append(passes, pngPass{rowBytes: rowBytes, rows: rows}) + } + + return passes, nil +} + +func pngRasterBytes(passes []pngPass) int64 { + var total int64 + + for _, pass := range passes { + row := (pass.rowBytes + 1) * pass.rows + if total > (1<<62)-row { + return 1<<62 - 1 + } + + total += row + } + + return total +} + +// validateRaster streams the inflated image data, checking every scanline +// filter byte and requiring exactly the IHDR-declared number of rows. It +// catches zlib streams that decompress cleanly but do not describe a valid +// raster (empty payloads, bogus filter bytes) and rejects extra data as a +// decompression bomb. +func validateRaster(compressed []byte, passes []pngPass) error { + source := bytes.NewReader(compressed) + + reader, err := zlib.NewReader(source) + if err != nil { + return corrupt("png idat zlib header", err) + } + + for _, pass := range passes { + for range pass.rows { + var filter [1]byte + if _, err := io.ReadFull(reader, filter[:]); err != nil { + return corrupt("png scanline filter", err) + } + + if filter[0] > 4 { + return corruptf("png invalid scanline filter %d", filter[0]) + } + + if _, err := io.CopyN(io.Discard, reader, pass.rowBytes); err != nil { + return corrupt("png scanline data", err) + } + } + } + + var extra [1]byte + switch _, err := io.ReadFull(reader, extra[:]); { + case errors.Is(err, io.EOF): + // A clean zlib EOF also verifies the Adler-32 trailer. bytes.Reader + // satisfies io.ByteReader, so the inflater consumes exactly the zlib + // stream; anything left is data after the stream, which PNG forbids. + if source.Len() != 0 { + return corruptf("png idat has %d trailing bytes", source.Len()) + } + + return nil + case err == nil: + return fmt.Errorf("%w: png raster exceeds declared dimensions", ErrTooLarge) + default: + return corrupt("png idat zlib trailer", err) } } func checkCriticalChunk(chunkType string, seen map[string]bool) error { if chunkType == "IHDR" || chunkType == "PLTE" || chunkType == "IDAT" || chunkType == "IEND" { seen[chunkType] = true + return nil } @@ -176,7 +471,7 @@ func checkCriticalChunk(chunkType string, seen map[string]bool) error { return nil } -func readChunkData(reader io.Reader, checksum io.Writer, length int64) ([]byte, error) { +func readChunkData(reader io.Reader, checksum io.Writer, length int64, sink io.Writer) ([]byte, error) { if length <= 64<<10 { data := make([]byte, length) if _, err := io.ReadFull(reader, data); err != nil { @@ -184,11 +479,19 @@ func readChunkData(reader io.Reader, checksum io.Writer, length int64) ([]byte, } _, _ = checksum.Write(data) + if sink != nil { + _, _ = sink.Write(data) + } return data, nil } - if _, err := io.CopyN(checksum, reader, length); err != nil { + destination := checksum + if sink != nil { + destination = io.MultiWriter(checksum, sink) + } + + if _, err := io.CopyN(destination, reader, length); err != nil { return nil, err } diff --git a/internal/imagecheck/imagecheck_test.go b/internal/imagecheck/imagecheck_test.go index b859ef6f..7d60e044 100644 --- a/internal/imagecheck/imagecheck_test.go +++ b/internal/imagecheck/imagecheck_test.go @@ -2,6 +2,9 @@ package imagecheck_test import ( "bytes" + "compress/zlib" + "encoding/binary" + "hash/crc32" "image" "image/color" "image/jpeg" @@ -45,6 +48,30 @@ func encodeJPEG(t *testing.T, width, height int) []byte { return buffer.Bytes() } +func pngChunk(ctype string, payload []byte) []byte { + out := make([]byte, 12+len(payload)) + binary.BigEndian.PutUint32(out[0:4], uint32(len(payload))) + copy(out[4:8], ctype) + copy(out[8:], payload) + + checksum := crc32.NewIEEE() + _, _ = checksum.Write(out[4 : 8+len(payload)]) + binary.BigEndian.PutUint32(out[8+len(payload):], checksum.Sum32()) + + return out +} + +func pngFile(chunks ...[]byte) []byte { + data := []byte(pngSignatureForTest) + for _, chunk := range chunks { + data = append(data, chunk...) + } + + return data +} + +const pngSignatureForTest = "\x89PNG\r\n\x1a\n" + func TestValidatePNG(t *testing.T) { info, err := imagecheck.Validate(bytes.NewReader(encodePNG(t, 32, 20)), 40_000_000) require.NoError(t, err) @@ -85,6 +112,352 @@ func TestValidateRejectsPNGCRCMismatch(t *testing.T) { require.ErrorIs(t, err, imagecheck.ErrCorrupt) } +// Builds a structurally valid PNG (signature, well-formed IHDR/IDAT/IEND, +// correct CRCs) whose IDAT payload is not a zlib stream at all. Merely a +// stream-like structure must not make the upload pass: it has to inflate. +func TestValidateRejectsPNGIDATNotAZlibStream(t *testing.T) { + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:4], 64) // width + binary.BigEndian.PutUint32(ihdr[4:8], 32) // height + ihdr[8] = 8 // bit depth + ihdr[9] = 6 // color type RGBA + + data := pngFile( + pngChunk("IHDR", ihdr), + pngChunk("IDAT", []byte("this is not a zlib stream, but the CRC is right")), + pngChunk("IEND", nil), + ) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func craftedPNG(t *testing.T, width, height int, interlace byte, raster []byte) []byte { + t.Helper() + + return craftedPNGType(t, width, height, 6, interlace, raster) +} + +func craftedPNGType(t *testing.T, width, height int, colorType, interlace byte, raster []byte) []byte { + t.Helper() + + return craftedPNGDepth(t, width, height, 8, colorType, interlace, raster) +} + +func craftedPNGDepth(t *testing.T, width, height int, bitDepth, colorType, interlace byte, raster []byte) []byte { + t.Helper() + + return pngFile( + pngChunk("IHDR", pngIHDR(width, height, bitDepth, colorType, interlace)), + pngChunk("IDAT", zlibBytes(t, raster)), + pngChunk("IEND", nil), + ) +} + +func pngIHDR(width, height int, bitDepth, colorType, interlace byte) []byte { + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:4], uint32(width)) + binary.BigEndian.PutUint32(ihdr[4:8], uint32(height)) + ihdr[8] = bitDepth + ihdr[9] = colorType + ihdr[12] = interlace + + return ihdr +} + +func zlibBytes(t *testing.T, raster []byte) []byte { + t.Helper() + + var compressed bytes.Buffer + + writer := zlib.NewWriter(&compressed) + _, err := writer.Write(raster) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + return compressed.Bytes() +} + +func TestValidateRejectsPNGEmptyRaster(t *testing.T) { + data := craftedPNG(t, 2, 2, 0, nil) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsPNGInvalidScanlineFilter(t *testing.T) { + raster := []byte{5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + data := craftedPNG(t, 2, 2, 0, raster) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsPNGPaletteWithoutPLTE(t *testing.T) { + raster := []byte{0, 0, 0, 0, 0, 0} // two rows of two 8-bit palette indexes + data := craftedPNGType(t, 2, 2, 3, 0, raster) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateAcceptsPalettePNG(t *testing.T) { + palette := color.Palette{color.RGBA{R: 255, A: 255}, color.RGBA{G: 255, A: 255}, color.RGBA{B: 255, A: 255}} + source := image.NewPaletted(image.Rect(0, 0, 13, 5), palette) + for x := range 13 { + for y := range 5 { + source.SetColorIndex(x, y, uint8((x+y)%len(palette))) + } + } + + var buffer bytes.Buffer + require.NoError(t, png.Encode(&buffer, source)) + + info, err := imagecheck.Validate(bytes.NewReader(buffer.Bytes()), 40_000_000) + require.NoError(t, err) + require.Equal(t, int64(13), info.Width) + require.Equal(t, int64(5), info.Height) +} + +func TestValidateRejectsPNGInvalidColorDepthCombination(t *testing.T) { + data := craftedPNGDepth(t, 1, 1, 1, 6, 0, []byte{0, 0, 0}) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsPNGDuplicateIHDR(t *testing.T) { + ihdr := pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)) + data := pngFile( + ihdr, + ihdr, + pngChunk("IDAT", zlibBytes(t, []byte{0, 0, 0, 0, 0})), + pngChunk("IEND", nil), + ) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsPNGDuplicatePLTE(t *testing.T) { + plte := pngChunk("PLTE", []byte{255, 0, 0}) + data := pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 3, 0)), + plte, + plte, + pngChunk("IDAT", zlibBytes(t, []byte{0, 0})), + pngChunk("IEND", nil), + ) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsPNGNonConsecutiveIDAT(t *testing.T) { + compressed := zlibBytes(t, []byte{0, 0, 0, 0, 0}) + middle := len(compressed) / 2 + + data := pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)), + pngChunk("IDAT", compressed[:middle]), + pngChunk("tEXt", []byte("k\x00v")), + pngChunk("IDAT", compressed[middle:]), + pngChunk("IEND", nil), + ) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsPNGTrailingIDATData(t *testing.T) { + idat := append(zlibBytes(t, []byte{0, 0, 0, 0, 0}), []byte("junk")...) + data := pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)), + pngChunk("IDAT", idat), + pngChunk("IEND", nil), + ) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateAcceptsSplitIDAT(t *testing.T) { + compressed := zlibBytes(t, []byte{0, 0, 0, 0, 0}) + middle := len(compressed) / 2 + + data := pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)), + pngChunk("IDAT", compressed[:middle]), + pngChunk("IDAT", compressed[middle:]), + pngChunk("IEND", nil), + ) + + info, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.NoError(t, err) + require.Equal(t, int64(1), info.Width) +} + +func TestValidateAcceptsStandardEncodings(t *testing.T) { + bounds := image.Rect(0, 0, 7, 5) + images := []image.Image{ + image.NewGray(bounds), + image.NewGray16(bounds), + image.NewRGBA(bounds), + image.NewNRGBA(bounds), + image.NewNRGBA64(bounds), + } + + for _, source := range images { + var buffer bytes.Buffer + require.NoError(t, png.Encode(&buffer, source)) + + info, err := imagecheck.Validate(bytes.NewReader(buffer.Bytes()), 40_000_000) + require.NoError(t, err) + require.Equal(t, int64(7), info.Width) + require.Equal(t, int64(5), info.Height) + } +} + +func TestValidateRejectsPNGPLTEEntriesExceedBitDepth(t *testing.T) { + data := pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 1, 3, 0)), + pngChunk("PLTE", []byte{255, 0, 0, 0, 255, 0, 0, 0, 255}), + pngChunk("IDAT", zlibBytes(t, []byte{0, 0})), + pngChunk("IEND", nil), + ) + + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt) +} + +func TestValidateRejectsInvalidTRNS(t *testing.T) { + rgb := pngChunk("IDAT", zlibBytes(t, []byte{0, 10, 20, 30})) + gray := pngChunk("IDAT", zlibBytes(t, []byte{0, 10})) + rgba := pngChunk("IDAT", zlibBytes(t, []byte{0, 10, 20, 30, 255})) + paletted := pngChunk("IDAT", zlibBytes(t, []byte{0, 0})) + + cases := map[string][]byte{ + "tRNS on RGBA": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)), + pngChunk("tRNS", []byte{0}), + rgba, + pngChunk("IEND", nil), + ), + "RGB length": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)), + pngChunk("tRNS", []byte{0}), + rgb, + pngChunk("IEND", nil), + ), + "gray length": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 0, 0)), + pngChunk("tRNS", []byte{0}), + gray, + pngChunk("IEND", nil), + ), + "palette before PLTE": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 3, 0)), + pngChunk("tRNS", []byte{0}), + pngChunk("PLTE", []byte{255, 0, 0, 0, 255, 0}), + paletted, + pngChunk("IEND", nil), + ), + "palette tRNS longer than PLTE": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 3, 0)), + pngChunk("PLTE", []byte{255, 0, 0}), + pngChunk("tRNS", []byte{0, 0}), + paletted, + pngChunk("IEND", nil), + ), + "duplicate tRNS": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)), + pngChunk("tRNS", make([]byte, 6)), + pngChunk("tRNS", make([]byte, 6)), + rgb, + pngChunk("IEND", nil), + ), + "tRNS after IDAT": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)), + rgb, + pngChunk("tRNS", make([]byte, 6)), + pngChunk("IEND", nil), + ), + } + + for name, data := range cases { + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.ErrorIs(t, err, imagecheck.ErrCorrupt, name) + } +} + +func TestValidateAcceptsValidTRNS(t *testing.T) { + cases := map[string][]byte{ + "RGB": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)), + pngChunk("tRNS", make([]byte, 6)), + pngChunk("IDAT", zlibBytes(t, []byte{0, 10, 20, 30})), + pngChunk("IEND", nil), + ), + "gray": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 8, 0, 0)), + pngChunk("tRNS", make([]byte, 2)), + pngChunk("IDAT", zlibBytes(t, []byte{0, 10})), + pngChunk("IEND", nil), + ), + "palette": pngFile( + pngChunk("IHDR", pngIHDR(1, 1, 1, 3, 0)), + pngChunk("PLTE", []byte{255, 0, 0, 0, 255, 0}), + pngChunk("tRNS", []byte{128}), + pngChunk("IDAT", zlibBytes(t, []byte{0, 0})), + pngChunk("IEND", nil), + ), + } + + for name, data := range cases { + _, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.NoError(t, err, name) + } +} + +// A valid Adam7 stream for a tall, narrow image exceeds the old +// (rowBytes+1)*(height+8) guess; the exact pass layout must accept it. +func TestValidateAcceptsAdam7PNG(t *testing.T) { + const width, height = 2, 256 + + var raster bytes.Buffer + + passes := [][4]int{ + {0, 0, 8, 8}, + {4, 0, 8, 8}, + {0, 4, 4, 8}, + {2, 0, 4, 4}, + {0, 2, 2, 4}, + {1, 0, 2, 2}, + {0, 1, 1, 2}, + } + for _, pass := range passes { + x0, y0, dx, dy := pass[0], pass[1], pass[2], pass[3] + if width <= x0 || height <= y0 { + continue + } + + for y := y0; y < height; y += dy { + raster.WriteByte(0) // filter: none + + for x := x0; x < width; x += dx { + raster.Write([]byte{byte(x), byte(y), 128, 255}) + } + } + } + + data := craftedPNG(t, width, height, 1, raster.Bytes()) + + info, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000) + require.NoError(t, err) + require.Equal(t, int64(width), info.Width) + require.Equal(t, int64(height), info.Height) +} + func TestValidateRejectsTruncatedJPEG(t *testing.T) { data := encodeJPEG(t, 64, 64) _, err := imagecheck.Validate(bytes.NewReader(data[:len(data)-32]), 40_000_000) diff --git a/internal/imageproc/exif.go b/internal/imageproc/exif.go new file mode 100644 index 00000000..943c0ed2 --- /dev/null +++ b/internal/imageproc/exif.go @@ -0,0 +1,118 @@ +package imageproc + +import ( + "encoding/binary" + "io" +) + +const ( + exifMaxBytes = 64 << 10 + exifTagOrient = 0x0112 +) + +// readEXIFOrientation extracts only IFD0 Orientation from a JPEG APP1 segment. +// The parser is deliberately small and bounded: no TIFF offsets are trusted and +// the whole segment is capped at 64 KiB, so attacker-controlled EXIF cannot make +// the worker allocate unbounded memory. +func readEXIFOrientation(r io.Reader) int { + data, err := io.ReadAll(io.LimitReader(r, exifMaxBytes)) + if err != nil || len(data) < 4 || data[0] != 0xff || data[1] != 0xd8 { + return 1 + } + + pos := 2 + + for pos+4 <= len(data) { + if data[pos] != 0xff { + return 1 + } + + marker := data[pos+1] + pos += 2 + + // Standalone markers without a length field. + if marker == 0xd8 || marker == 0xd9 || marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7) { + continue + } + + if pos+2 > len(data) { + return 1 + } + + length := int(binary.BigEndian.Uint16(data[pos : pos+2])) + if length < 2 || pos+length > len(data) { + return 1 + } + + segment := data[pos+2 : pos+length] + pos += length + + if marker != 0xe1 || len(segment) < 6 { + continue + } + + if string(segment[:6]) != "Exif\x00\x00" { + continue + } + + return parseTIFFOrientation(segment[6:]) + } + + return 1 +} + +func parseTIFFOrientation(tiff []byte) int { + if len(tiff) < 8 { + return 1 + } + + var order binary.ByteOrder + + switch string(tiff[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 1 + } + + if order.Uint16(tiff[2:4]) != 42 { + return 1 + } + + ifdOffset := int(order.Uint32(tiff[4:8])) + if ifdOffset < 8 || ifdOffset+2 > len(tiff) { + return 1 + } + + count := int(order.Uint16(tiff[ifdOffset : ifdOffset+2])) + if count > 512 { + count = 512 + } + + entry := ifdOffset + 2 + + for range count { + if entry+12 > len(tiff) { + return 1 + } + + tag := order.Uint16(tiff[entry : entry+2]) + typ := order.Uint16(tiff[entry+2 : entry+4]) + n := order.Uint32(tiff[entry+4 : entry+8]) + + if tag == exifTagOrient && typ == 3 && n == 1 { + value := order.Uint16(tiff[entry+8 : entry+10]) + if value >= 1 && value <= 8 { + return int(value) + } + + return 1 + } + + entry += 12 + } + + return 1 +} diff --git a/internal/imageproc/imageproc.go b/internal/imageproc/imageproc.go new file mode 100644 index 00000000..545b16e4 --- /dev/null +++ b/internal/imageproc/imageproc.go @@ -0,0 +1,259 @@ +// Package imageproc turns one decoded original into the contract variants. +// Output bytes are deterministic for a given pipeline version, source bytes and +// encoder versions; the worker uses that property for idempotent retries. +package imageproc + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "image" + "image/color" + "image/jpeg" + _ "image/png" // register the PNG decoder + "io" + "math" + "os" + + "github.com/gen2brain/webp" + xdraw "golang.org/x/image/draw" + + "mtusi-messenger/internal/domain" +) + +const ( + thumbnailMax = 256 + cropSize = 128 + optimizedMax = 1280 + defaultMethod = 2 +) + +// Params are immutable per pipeline version. They are compiled into the binary +// so a retry can never silently produce different bytes than the first attempt. +type Params struct { + JPEGQuality int + WebPQuality int + WebPMethod int +} + +// Pipeline returns the versioned encoder parameters. Only v1 and v2 are +// defined; the course baseline is v1. +func Pipeline(version int) (Params, bool) { + switch version { + case 1: + return Params{JPEGQuality: 85, WebPQuality: 80, WebPMethod: defaultMethod}, true + case 2: + return Params{JPEGQuality: 70, WebPQuality: 65, WebPMethod: 4}, true + default: + return Params{}, false + } +} + +// Image is one encoded variant. +type Image struct { + Name domain.VariantName + ContentType string + Data []byte + SizeBytes int64 + Width int + Height int + SHA256 string +} + +// Load decodes a PNG/JPEG file, applies EXIF orientation and composites alpha +// on white. The returned image is always *image.RGBA, which bounds the variant +// encoders to 8-bit output. +func Load(path string, maxPixels int64) (image.Image, string, error) { + file, err := os.Open(path) + if err != nil { + return nil, "", fmt.Errorf("open source: %w", err) + } + defer file.Close() + + cfg, format, err := image.DecodeConfig(file) + if err != nil { + return nil, "", fmt.Errorf("decode config: %w", err) + } + + if cfg.Width <= 0 || cfg.Height <= 0 { + return nil, "", fmt.Errorf("invalid dimensions %dx%d", cfg.Width, cfg.Height) + } + + if maxPixels > 0 && int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return nil, "", fmt.Errorf("image has %d pixels, limit %d", int64(cfg.Width)*int64(cfg.Height), maxPixels) + } + + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, "", fmt.Errorf("rewind source: %w", err) + } + + decoded, _, err := image.Decode(file) + if err != nil { + return nil, "", fmt.Errorf("decode image: %w", err) + } + + orientation := 1 + if format == "jpeg" { + if _, err := file.Seek(0, io.SeekStart); err == nil { + orientation = readEXIFOrientation(file) + } + } + + return toWhiteRGBA(decoded, orientation), format, nil +} + +func toWhiteRGBA(src image.Image, orientation int) image.Image { + transformed := orient(src, orientation) + bounds := transformed.Bounds() + + dst := image.NewRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + xdraw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, xdraw.Src) + xdraw.Draw(dst, dst.Bounds(), transformed, bounds.Min, xdraw.Over) + + return dst +} + +// orient applies the eight EXIF orientation values. The result has the same +// bounds as the source except for 90-degree rotations. +func orient(src image.Image, orientation int) image.Image { + if orientation < 2 || orientation > 8 { + return src + } + + b := src.Bounds() + w, h := b.Dx(), b.Dy() + + width, height := w, h + if orientation >= 5 { + width, height = h, w + } + + dst := image.NewRGBA(image.Rect(0, 0, width, height)) + + for y := range h { + for x := range w { + nx, ny := orientPoint(orientation, x, y, w, h) + + dst.Set(nx, ny, src.At(b.Min.X+x, b.Min.Y+y)) + } + } + + return dst +} + +func orientPoint(orientation, x, y, w, h int) (int, int) { + switch orientation { + case 2: + return w - 1 - x, y + case 3: + return w - 1 - x, h - 1 - y + case 4: + return x, h - 1 - y + case 5: + return y, x + case 6: + return h - 1 - y, x + case 7: + return h - 1 - y, w - 1 - x + case 8: + return y, w - 1 - x + default: + return x, y + } +} + +// Thumbnail fits inside 256x256 without upscaling; dimensions are floored and +// never below 1 px. +func Thumbnail(src image.Image, params Params) (Image, error) { + return encodeJPEG(fit(src, thumbnailMax, thumbnailMax, false), domain.VariantThumbnail, params) +} + +// Crop takes the centered square and scales it to 128x128. Upscaling small +// sources is allowed by the contract. +func Crop(src image.Image, params Params) (Image, error) { + b := src.Bounds() + side := b.Dx() + if b.Dy() < side { + side = b.Dy() + } + + left := b.Min.X + (b.Dx()-side)/2 + top := b.Min.Y + (b.Dy()-side)/2 + + square := image.NewRGBA(image.Rect(0, 0, side, side)) + xdraw.Draw(square, square.Bounds(), src, image.Pt(left, top), xdraw.Src) + + dst := image.NewRGBA(image.Rect(0, 0, cropSize, cropSize)) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), square, square.Bounds(), xdraw.Over, nil) + + return encodeJPEG(dst, domain.VariantCrop, params) +} + +// Optimized fits inside 1280x1280 without upscaling and encodes WebP. Metadata +// is stripped because the output is a fresh re-encode. +func Optimized(src image.Image, params Params) (Image, error) { + dst := fit(src, optimizedMax, optimizedMax, false) + + var buf bytes.Buffer + + if err := webp.Encode(&buf, dst, webp.Options{ + Quality: params.WebPQuality, + Method: params.WebPMethod, + }); err != nil { + return Image{}, fmt.Errorf("encode webp: %w", err) + } + + return newImage(domain.VariantOptimized, "image/webp", buf.Bytes(), dst.Bounds().Dx(), dst.Bounds().Dy()), nil +} + +func encodeJPEG(dst image.Image, name domain.VariantName, params Params) (Image, error) { + var buf bytes.Buffer + + if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: params.JPEGQuality}); err != nil { + return Image{}, fmt.Errorf("encode jpeg: %w", err) + } + + return newImage(name, "image/jpeg", buf.Bytes(), dst.Bounds().Dx(), dst.Bounds().Dy()), nil +} + +func newImage(name domain.VariantName, contentType string, data []byte, width, height int) Image { + sum := sha256.Sum256(data) + + return Image{ + Name: name, + ContentType: contentType, + Data: data, + SizeBytes: int64(len(data)), + Width: width, + Height: height, + SHA256: hex.EncodeToString(sum[:]), + } +} + +// fit scales src into maxWidth x maxHeight preserving aspect ratio, floors the +// dimensions, enforces at least 1 px and, unless allowUpscale is set, never +// enlarges the source. +func fit(src image.Image, maxWidth, maxHeight int, allowUpscale bool) *image.RGBA { + b := src.Bounds() + w, h := b.Dx(), b.Dy() + + scale := math.Min(float64(maxWidth)/float64(w), float64(maxHeight)/float64(h)) + if !allowUpscale && scale > 1 { + scale = 1 + } + + width := int(math.Floor(float64(w) * scale)) + height := int(math.Floor(float64(h) * scale)) + if width < 1 { + width = 1 + } + if height < 1 { + height = 1 + } + + dst := image.NewRGBA(image.Rect(0, 0, width, height)) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, b, xdraw.Over, nil) + + return dst +} diff --git a/internal/imageproc/imageproc_test.go b/internal/imageproc/imageproc_test.go new file mode 100644 index 00000000..24b1002b --- /dev/null +++ b/internal/imageproc/imageproc_test.go @@ -0,0 +1,222 @@ +package imageproc_test + +import ( + "bytes" + "encoding/binary" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "path/filepath" + "testing" + + "github.com/gen2brain/webp" + "github.com/stretchr/testify/require" + + "mtusi-messenger/internal/domain" + "mtusi-messenger/internal/imageproc" +) + +func writePNG(t *testing.T, width, height int, alpha bool) string { + t.Helper() + + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for x := range width { + for y := range height { + a := uint8(255) + if alpha && x < width/2 && y < height/2 { + a = 0 + } + img.Set(x, y, color.NRGBA{R: 200, G: 40, B: 20, A: a}) + } + } + + var buf bytes.Buffer + require.NoError(t, png.Encode(&buf, img)) + + path := filepath.Join(t.TempDir(), "source.png") + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o600)) + + return path +} + +func writeJPEG(t *testing.T, width, height, orientation int) string { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for x := range width { + for y := range height { + img.Set(x, y, color.RGBA{R: 200, G: 40, B: 20, A: 255}) + } + } + + var raw bytes.Buffer + require.NoError(t, jpeg.Encode(&raw, img, &jpeg.Options{Quality: 95})) + + data := append([]byte(nil), raw.Bytes()...) + if orientation > 1 { + data = insertEXIFOrientation(t, data, uint16(orientation)) + } + + path := filepath.Join(t.TempDir(), "source.jpg") + require.NoError(t, os.WriteFile(path, data, 0o600)) + + return path +} + +func insertEXIFOrientation(t *testing.T, jpegBytes []byte, orientation uint16) []byte { + t.Helper() + + // TIFF: II, 42, IFD0 at offset 8, one entry (0x0112 SHORT 1 = orientation). + tiff := make([]byte, 8+2+12+4) + binary.LittleEndian.PutUint16(tiff[0:2], 0x4949) + binary.LittleEndian.PutUint16(tiff[2:4], 42) + binary.LittleEndian.PutUint32(tiff[4:8], 8) + binary.LittleEndian.PutUint16(tiff[8:10], 1) + binary.LittleEndian.PutUint16(tiff[10:12], 0x0112) + binary.LittleEndian.PutUint16(tiff[12:14], 3) + binary.LittleEndian.PutUint32(tiff[14:18], 1) + binary.LittleEndian.PutUint16(tiff[18:20], orientation) + + payload := append([]byte("Exif\x00\x00"), tiff...) + segment := make([]byte, 4+len(payload)) + segment[0] = 0xff + segment[1] = 0xe1 + binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2)) + copy(segment[4:], payload) + + // Insert immediately after SOI (before the first DQT/SOF). + out := append([]byte(nil), jpegBytes[:2]...) + out = append(out, segment...) + out = append(out, jpegBytes[2:]...) + + return out +} + +func decodeConfig(t *testing.T, data []byte) image.Config { + t.Helper() + + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + require.NoError(t, err) + + return cfg +} + +func TestThumbnailContract(t *testing.T) { + params, ok := imageproc.Pipeline(1) + require.True(t, ok) + + cases := []struct { + name string + width int + height int + expected [2]int + }{ + {"320x200", 320, 200, [2]int{256, 160}}, + {"no upscale 64x64", 64, 64, [2]int{64, 64}}, + {"no upscale 64x40", 64, 40, [2]int{64, 40}}, + {"floor 1000x10", 1000, 10, [2]int{256, 2}}, + {"height limited", 100, 320, [2]int{80, 256}}, + {"minimum 1x1", 1, 1, [2]int{1, 1}}, + } + + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + img, _, err := imageproc.Load(writePNG(t, test.width, test.height, false), 40_000_000) + require.NoError(t, err) + + thumb, err := imageproc.Thumbnail(img, params) + require.NoError(t, err) + + cfg := decodeConfig(t, thumb.Data) + require.Equal(t, test.expected[0], cfg.Width) + require.Equal(t, test.expected[1], cfg.Height) + require.Equal(t, test.expected[0], thumb.Width) + require.Equal(t, test.expected[1], thumb.Height) + require.Equal(t, "image/jpeg", thumb.ContentType) + require.NotContains(t, string(thumb.Data), "Exif") + }) + } +} + +func TestCropAndOptimized(t *testing.T) { + params, ok := imageproc.Pipeline(1) + require.True(t, ok) + + img, _, err := imageproc.Load(writePNG(t, 320, 200, false), 40_000_000) + require.NoError(t, err) + + crop, err := imageproc.Crop(img, params) + require.NoError(t, err) + cfg := decodeConfig(t, crop.Data) + require.Equal(t, 128, cfg.Width) + require.Equal(t, 128, cfg.Height) + + optimized, err := imageproc.Optimized(img, params) + require.NoError(t, err) + cfg = decodeConfig(t, optimized.Data) + require.Equal(t, 320, cfg.Width) + require.Equal(t, 200, cfg.Height) + require.Equal(t, "image/webp", optimized.ContentType) + + decoded, err := webp.Decode(bytes.NewReader(optimized.Data)) + require.NoError(t, err) + require.Equal(t, 320, decoded.Bounds().Dx()) + require.Equal(t, 200, decoded.Bounds().Dy()) +} + +func TestEXIFOrientationChangesThumbnailDimensions(t *testing.T) { + params, ok := imageproc.Pipeline(1) + require.True(t, ok) + + for _, orientation := range []int{6, 8} { + img, _, err := imageproc.Load(writeJPEG(t, 320, 200, orientation), 40_000_000) + require.NoError(t, err) + + thumb, err := imageproc.Thumbnail(img, params) + require.NoError(t, err) + + cfg := decodeConfig(t, thumb.Data) + require.Equal(t, 160, cfg.Width, "orientation %d", orientation) + require.Equal(t, 256, cfg.Height, "orientation %d", orientation) + } +} + +func TestAlphaCompositedOnWhite(t *testing.T) { + params, ok := imageproc.Pipeline(1) + require.True(t, ok) + + img, _, err := imageproc.Load(writePNG(t, 100, 100, true), 40_000_000) + require.NoError(t, err) + + thumb, err := imageproc.Thumbnail(img, params) + require.NoError(t, err) + + decoded, err := jpeg.Decode(bytes.NewReader(thumb.Data)) + require.NoError(t, err) + + r, g, b, _ := decoded.At(0, 0).RGBA() + require.Greater(t, r, uint32(0xf000), "transparent corner must be white") + require.Greater(t, g, uint32(0xf000), "transparent corner must be white") + require.Greater(t, b, uint32(0xf000), "transparent corner must be white") +} + +func TestPipelineVersionsDiffer(t *testing.T) { + v1, ok := imageproc.Pipeline(1) + require.True(t, ok) + v2, ok := imageproc.Pipeline(2) + require.True(t, ok) + require.NotEqual(t, v1, v2) + + _, ok = imageproc.Pipeline(3) + require.False(t, ok) +} + +func TestVariantNames(t *testing.T) { + require.Equal(t, "image/jpeg", domain.VariantThumbnail.ContentType()) + require.Equal(t, "image/jpeg", domain.VariantCrop.ContentType()) + require.Equal(t, "image/webp", domain.VariantOptimized.ContentType()) + require.Equal(t, "thumbnail.jpg", domain.VariantThumbnail.ObjectName()) + require.Equal(t, "optimized.webp", domain.VariantOptimized.ObjectName()) +} diff --git a/internal/outbox/relay.go b/internal/outbox/relay.go new file mode 100644 index 00000000..4df7423e --- /dev/null +++ b/internal/outbox/relay.go @@ -0,0 +1,168 @@ +// Package outbox publishes the durable job intents with publisher confirms and +// reclaims abandoned relay claims and expired worker leases. +package outbox + +import ( + "context" + "log/slog" + "time" + + "mtusi-messenger/internal/domain" + "mtusi-messenger/internal/failpoint" +) + +type Repository interface { + ClaimOutbox(ctx context.Context, owner string, claimTTL time.Duration, limit int) ([]domain.OutboxMessage, error) + MarkPublished(ctx context.Context, ids []int64) error + ReleaseOutbox(ctx context.Context, id int64, reason string) error + ReapOutboxClaims(ctx context.Context) error + ReapExpiredLeases(ctx context.Context, limit int) (int, error) + ReconcileQueuedJobs(ctx context.Context, olderThan time.Duration, limit int) (int, error) + PruneOutbox(ctx context.Context, retention time.Duration) error + OldestUnpublished(ctx context.Context) (time.Time, error) +} + +type Publisher interface { + PublishConfirmed(ctx context.Context, routingKey string, message domain.JobMessage) error +} + +type Config struct { + Owner string + PollInterval time.Duration + ClaimTTL time.Duration + Batch int + ReaperInterval time.Duration + ReconcileAfter time.Duration + Retention time.Duration +} + +type Relay struct { + repo Repository + publisher Publisher + config Config + log *slog.Logger + + lastPublished time.Time + lastReap time.Time +} + +func New(repo Repository, publisher Publisher, config Config, log *slog.Logger) *Relay { + if log == nil { + log = slog.Default() + } + + return &Relay{repo: repo, publisher: publisher, config: config, log: log} +} + +func (r *Relay) Run(ctx context.Context) { + poll := time.NewTicker(r.config.PollInterval) + defer poll.Stop() + + reaper := time.NewTicker(r.config.ReaperInterval) + defer reaper.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-poll.C: + if _, err := r.RunOnce(ctx); err != nil { + r.log.WarnContext(ctx, "outbox publish cycle failed", slog.Any("error", err)) + } + case <-reaper.C: + r.reap(ctx) + } + } +} + +// RunOnce claims due rows, publishes each with a broker confirm and marks only +// confirmed rows. A crash between confirm and mark causes a duplicate publish, +// which the worker tolerates by CAS. +func (r *Relay) RunOnce(ctx context.Context) (int, error) { + messages, err := r.repo.ClaimOutbox(ctx, r.config.Owner, r.config.ClaimTTL, r.config.Batch) + if err != nil { + return 0, err + } + + published := make([]int64, 0, len(messages)) + + for _, message := range messages { + job, err := domain.ParseJobMessage(message.Payload) + if err != nil { + // A malformed payload is an application bug, not a transient + // failure. Keep the row claim until it expires and log loudly; the + // operator can inspect the outbox. + r.log.ErrorContext(ctx, "outbox payload is invalid", + slog.Int64("outbox_id", message.ID), + slog.Any("error", err), + ) + _ = r.repo.ReleaseOutbox(ctx, message.ID, "invalid payload") + + continue + } + + failpoint.Check("OUTBOX_AFTER_COMMIT", job.JobID) + + if err := r.publisher.PublishConfirmed(ctx, message.RoutingKey, job); err != nil { + _ = r.repo.ReleaseOutbox(ctx, message.ID, err.Error()) + + continue + } + + published = append(published, message.ID) + r.lastPublished = time.Now() + } + + if err := r.repo.MarkPublished(ctx, published); err != nil { + return len(published), err + } + + return len(published), nil +} + +func (r *Relay) reap(ctx context.Context) { + if err := r.repo.ReapOutboxClaims(ctx); err != nil { + r.log.WarnContext(ctx, "outbox claim reaper failed", slog.Any("error", err)) + } + + reaped, err := r.repo.ReapExpiredLeases(ctx, 100) + if err != nil { + r.log.WarnContext(ctx, "lease reaper failed", slog.Any("error", err)) + } else if reaped > 0 { + r.log.InfoContext(ctx, "expired job leases requeued", slog.Int("count", reaped)) + } + + // The broker is not the source of truth. If a delivery was dead-lettered + // or lost, the queued job has no unpublished outbox row; republish it. + if r.config.ReconcileAfter > 0 { + reconciled, err := r.repo.ReconcileQueuedJobs(ctx, r.config.ReconcileAfter, 100) + if err != nil { + r.log.WarnContext(ctx, "queued-job reconciliation failed", slog.Any("error", err)) + } else if reconciled > 0 { + r.log.InfoContext(ctx, "lost queued jobs reconciled", slog.Int("count", reconciled)) + } + } + + if r.config.Retention > 0 { + if err := r.repo.PruneOutbox(ctx, r.config.Retention); err != nil { + r.log.WarnContext(ctx, "outbox retention failed", slog.Any("error", err)) + } + } + + r.lastReap = time.Now() +} + +// Health returns the age of the oldest unpublished intent. A bounded age is a +// better relay readiness signal than "connected to broker". +func (r *Relay) Health(ctx context.Context) (oldest time.Duration, err error) { + oldestAt, err := r.repo.OldestUnpublished(ctx) + if err != nil { + return 0, err + } + + if oldestAt.IsZero() { + return 0, nil + } + + return time.Since(oldestAt), nil +} diff --git a/internal/platform/objectstore/objectstore.go b/internal/platform/objectstore/objectstore.go index 8d5aa62c..7a75726b 100644 --- a/internal/platform/objectstore/objectstore.go +++ b/internal/platform/objectstore/objectstore.go @@ -22,6 +22,8 @@ type Object struct { // because AWS SigV4 may need to rewind the payload; the proxy upload path always // passes an *os.File. type Store interface { + // Ping verifies that the bucket is reachable and the credentials are valid. + Ping(ctx context.Context) error Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string, metadata map[string]string) error Open(ctx context.Context, key string) (io.ReadCloser, Object, error) Head(ctx context.Context, key string) (Object, error) diff --git a/internal/platform/objectstore/s3.go b/internal/platform/objectstore/s3.go index ff708c66..f21c31c0 100644 --- a/internal/platform/objectstore/s3.go +++ b/internal/platform/objectstore/s3.go @@ -91,6 +91,19 @@ func New(cfg Config) (*S3, error) { return st, nil } +func (s *S3) Ping(ctx context.Context) error { + // HeadBucket needs bucket-level permissions that the application identity + // intentionally does not have. A HeadObject on a sentinel staging key is + // enough to prove connectivity and the object-level credentials; a missing + // sentinel is the expected healthy result. + _, err := s.Head(ctx, "staging/.health/ping") + if err == nil || errors.Is(err, domain.ErrNotFound) { + return nil + } + + return err +} + func (s *S3) Put( ctx context.Context, key string, diff --git a/internal/service/attachment/mocks/Repository.go b/internal/service/attachment/mocks/Repository.go index ac27f08d..f31bea67 100644 --- a/internal/service/attachment/mocks/Repository.go +++ b/internal/service/attachment/mocks/Repository.go @@ -7,7 +7,6 @@ package mocks import ( "context" "mtusi-messenger/internal/domain" - "time" "github.com/google/uuid" mock "github.com/stretchr/testify/mock" @@ -49,53 +48,65 @@ func (_m *MockRepository) EXPECT() *MockRepository_Expecter { return &MockRepository_Expecter{mock: &_m.Mock} } -// Create provides a mock function for the type MockRepository -func (_mock *MockRepository) Create(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error) { - ret := _mock.Called(ctx, draft) +// Admission provides a mock function for the type MockRepository +func (_mock *MockRepository) Admission(ctx context.Context, ownerID uuid.UUID) (int, int, int64, error) { + ret := _mock.Called(ctx, ownerID) if len(ret) == 0 { - panic("no return value specified for Create") + panic("no return value specified for Admission") } - var r0 domain.Attachment - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft) (domain.Attachment, error)); ok { - return returnFunc(ctx, draft) + var r0 int + var r1 int + var r2 int64 + var r3 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID) (int, int, int64, error)); ok { + return returnFunc(ctx, ownerID) } - if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft) domain.Attachment); ok { - r0 = returnFunc(ctx, draft) + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID) int); ok { + r0 = returnFunc(ctx, ownerID) } else { - r0 = ret.Get(0).(domain.Attachment) + r0 = ret.Get(0).(int) } - if returnFunc, ok := ret.Get(1).(func(context.Context, domain.AttachmentDraft) error); ok { - r1 = returnFunc(ctx, draft) + if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID) int); ok { + r1 = returnFunc(ctx, ownerID) } else { - r1 = ret.Error(1) + r1 = ret.Get(1).(int) } - return r0, r1 + if returnFunc, ok := ret.Get(2).(func(context.Context, uuid.UUID) int64); ok { + r2 = returnFunc(ctx, ownerID) + } else { + r2 = ret.Get(2).(int64) + } + if returnFunc, ok := ret.Get(3).(func(context.Context, uuid.UUID) error); ok { + r3 = returnFunc(ctx, ownerID) + } else { + r3 = ret.Error(3) + } + return r0, r1, r2, r3 } -// MockRepository_Create_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Create' -type MockRepository_Create_Call struct { +// MockRepository_Admission_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Admission' +type MockRepository_Admission_Call struct { *mock.Call } -// Create is a helper method to define mock.On call +// Admission is a helper method to define mock.On call // - ctx context.Context -// - draft domain.AttachmentDraft -func (_e *MockRepository_Expecter) Create(ctx any, draft any) *MockRepository_Create_Call { - return &MockRepository_Create_Call{Call: _e.mock.On("Create", ctx, draft)} +// - ownerID uuid.UUID +func (_e *MockRepository_Expecter) Admission(ctx any, ownerID any) *MockRepository_Admission_Call { + return &MockRepository_Admission_Call{Call: _e.mock.On("Admission", ctx, ownerID)} } -func (_c *MockRepository_Create_Call) Run(run func(ctx context.Context, draft domain.AttachmentDraft)) *MockRepository_Create_Call { +func (_c *MockRepository_Admission_Call) Run(run func(ctx context.Context, ownerID uuid.UUID)) *MockRepository_Admission_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 domain.AttachmentDraft + var arg1 uuid.UUID if args[1] != nil { - arg1 = args[1].(domain.AttachmentDraft) + arg1 = args[1].(uuid.UUID) } run( arg0, @@ -105,12 +116,264 @@ func (_c *MockRepository_Create_Call) Run(run func(ctx context.Context, draft do return _c } -func (_c *MockRepository_Create_Call) Return(attachment domain.Attachment, err error) *MockRepository_Create_Call { +func (_c *MockRepository_Admission_Call) Return(userPending int, totalPending int, stagedBytes int64, err error) *MockRepository_Admission_Call { + _c.Call.Return(userPending, totalPending, stagedBytes, err) + return _c +} + +func (_c *MockRepository_Admission_Call) RunAndReturn(run func(ctx context.Context, ownerID uuid.UUID) (int, int, int64, error)) *MockRepository_Admission_Call { + _c.Call.Return(run) + return _c +} + +// CompletePresign provides a mock function for the type MockRepository +func (_mock *MockRepository) CompletePresign(ctx context.Context, ownerID uuid.UUID, attachmentID uuid.UUID, verifiedKey string, finalKey string, pipelineVersion int, maxAttempts int, jobID uuid.UUID) (domain.Attachment, error) { + ret := _mock.Called(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID) + + if len(ret) == 0 { + panic("no return value specified for CompletePresign") + } + + var r0 domain.Attachment + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, string, string, int, int, uuid.UUID) (domain.Attachment, error)); ok { + return returnFunc(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, string, string, int, int, uuid.UUID) domain.Attachment); ok { + r0 = returnFunc(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID) + } else { + r0 = ret.Get(0).(domain.Attachment) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, uuid.UUID, string, string, int, int, uuid.UUID) error); ok { + r1 = returnFunc(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockRepository_CompletePresign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CompletePresign' +type MockRepository_CompletePresign_Call struct { + *mock.Call +} + +// CompletePresign is a helper method to define mock.On call +// - ctx context.Context +// - ownerID uuid.UUID +// - attachmentID uuid.UUID +// - verifiedKey string +// - finalKey string +// - pipelineVersion int +// - maxAttempts int +// - jobID uuid.UUID +func (_e *MockRepository_Expecter) CompletePresign(ctx any, ownerID any, attachmentID any, verifiedKey any, finalKey any, pipelineVersion any, maxAttempts any, jobID any) *MockRepository_CompletePresign_Call { + return &MockRepository_CompletePresign_Call{Call: _e.mock.On("CompletePresign", ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID)} +} + +func (_c *MockRepository_CompletePresign_Call) Run(run func(ctx context.Context, ownerID uuid.UUID, attachmentID uuid.UUID, verifiedKey string, finalKey string, pipelineVersion int, maxAttempts int, jobID uuid.UUID)) *MockRepository_CompletePresign_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uuid.UUID + if args[1] != nil { + arg1 = args[1].(uuid.UUID) + } + var arg2 uuid.UUID + if args[2] != nil { + arg2 = args[2].(uuid.UUID) + } + var arg3 string + if args[3] != nil { + arg3 = args[3].(string) + } + var arg4 string + if args[4] != nil { + arg4 = args[4].(string) + } + var arg5 int + if args[5] != nil { + arg5 = args[5].(int) + } + var arg6 int + if args[6] != nil { + arg6 = args[6].(int) + } + var arg7 uuid.UUID + if args[7] != nil { + arg7 = args[7].(uuid.UUID) + } + run( + arg0, + arg1, + arg2, + arg3, + arg4, + arg5, + arg6, + arg7, + ) + }) + return _c +} + +func (_c *MockRepository_CompletePresign_Call) Return(attachment domain.Attachment, err error) *MockRepository_CompletePresign_Call { _c.Call.Return(attachment, err) return _c } -func (_c *MockRepository_Create_Call) RunAndReturn(run func(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error)) *MockRepository_Create_Call { +func (_c *MockRepository_CompletePresign_Call) RunAndReturn(run func(ctx context.Context, ownerID uuid.UUID, attachmentID uuid.UUID, verifiedKey string, finalKey string, pipelineVersion int, maxAttempts int, jobID uuid.UUID) (domain.Attachment, error)) *MockRepository_CompletePresign_Call { + _c.Call.Return(run) + return _c +} + +// CreatePresign provides a mock function for the type MockRepository +func (_mock *MockRepository) CreatePresign(ctx context.Context, draft domain.AttachmentDraft, limits domain.AdmissionLimits) (domain.Attachment, error) { + ret := _mock.Called(ctx, draft, limits) + + if len(ret) == 0 { + panic("no return value specified for CreatePresign") + } + + var r0 domain.Attachment + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.AdmissionLimits) (domain.Attachment, error)); ok { + return returnFunc(ctx, draft, limits) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.AdmissionLimits) domain.Attachment); ok { + r0 = returnFunc(ctx, draft, limits) + } else { + r0 = ret.Get(0).(domain.Attachment) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, domain.AttachmentDraft, domain.AdmissionLimits) error); ok { + r1 = returnFunc(ctx, draft, limits) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockRepository_CreatePresign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreatePresign' +type MockRepository_CreatePresign_Call struct { + *mock.Call +} + +// CreatePresign is a helper method to define mock.On call +// - ctx context.Context +// - draft domain.AttachmentDraft +// - limits domain.AdmissionLimits +func (_e *MockRepository_Expecter) CreatePresign(ctx any, draft any, limits any) *MockRepository_CreatePresign_Call { + return &MockRepository_CreatePresign_Call{Call: _e.mock.On("CreatePresign", ctx, draft, limits)} +} + +func (_c *MockRepository_CreatePresign_Call) Run(run func(ctx context.Context, draft domain.AttachmentDraft, limits domain.AdmissionLimits)) *MockRepository_CreatePresign_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 domain.AttachmentDraft + if args[1] != nil { + arg1 = args[1].(domain.AttachmentDraft) + } + var arg2 domain.AdmissionLimits + if args[2] != nil { + arg2 = args[2].(domain.AdmissionLimits) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockRepository_CreatePresign_Call) Return(attachment domain.Attachment, err error) *MockRepository_CreatePresign_Call { + _c.Call.Return(attachment, err) + return _c +} + +func (_c *MockRepository_CreatePresign_Call) RunAndReturn(run func(ctx context.Context, draft domain.AttachmentDraft, limits domain.AdmissionLimits) (domain.Attachment, error)) *MockRepository_CreatePresign_Call { + _c.Call.Return(run) + return _c +} + +// CreateQueued provides a mock function for the type MockRepository +func (_mock *MockRepository) CreateQueued(ctx context.Context, draft domain.AttachmentDraft, job domain.Job, limits domain.AdmissionLimits) (domain.Attachment, error) { + ret := _mock.Called(ctx, draft, job, limits) + + if len(ret) == 0 { + panic("no return value specified for CreateQueued") + } + + var r0 domain.Attachment + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.Job, domain.AdmissionLimits) (domain.Attachment, error)); ok { + return returnFunc(ctx, draft, job, limits) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.Job, domain.AdmissionLimits) domain.Attachment); ok { + r0 = returnFunc(ctx, draft, job, limits) + } else { + r0 = ret.Get(0).(domain.Attachment) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, domain.AttachmentDraft, domain.Job, domain.AdmissionLimits) error); ok { + r1 = returnFunc(ctx, draft, job, limits) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockRepository_CreateQueued_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateQueued' +type MockRepository_CreateQueued_Call struct { + *mock.Call +} + +// CreateQueued is a helper method to define mock.On call +// - ctx context.Context +// - draft domain.AttachmentDraft +// - job domain.Job +// - limits domain.AdmissionLimits +func (_e *MockRepository_Expecter) CreateQueued(ctx any, draft any, job any, limits any) *MockRepository_CreateQueued_Call { + return &MockRepository_CreateQueued_Call{Call: _e.mock.On("CreateQueued", ctx, draft, job, limits)} +} + +func (_c *MockRepository_CreateQueued_Call) Run(run func(ctx context.Context, draft domain.AttachmentDraft, job domain.Job, limits domain.AdmissionLimits)) *MockRepository_CreateQueued_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 domain.AttachmentDraft + if args[1] != nil { + arg1 = args[1].(domain.AttachmentDraft) + } + var arg2 domain.Job + if args[2] != nil { + arg2 = args[2].(domain.Job) + } + var arg3 domain.AdmissionLimits + if args[3] != nil { + arg3 = args[3].(domain.AdmissionLimits) + } + run( + arg0, + arg1, + arg2, + arg3, + ) + }) + return _c +} + +func (_c *MockRepository_CreateQueued_Call) Return(attachment domain.Attachment, err error) *MockRepository_CreateQueued_Call { + _c.Call.Return(attachment, err) + return _c +} + +func (_c *MockRepository_CreateQueued_Call) RunAndReturn(run func(ctx context.Context, draft domain.AttachmentDraft, job domain.Job, limits domain.AdmissionLimits) (domain.Attachment, error)) *MockRepository_CreateQueued_Call { _c.Call.Return(run) return _c } @@ -316,6 +579,84 @@ func (_c *MockRepository_GetForOwner_Call) RunAndReturn(run func(ctx context.Con return _c } +// GetVariant provides a mock function for the type MockRepository +func (_mock *MockRepository) GetVariant(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int, name domain.VariantName) (domain.Variant, error) { + ret := _mock.Called(ctx, attachmentID, pipelineVersion, name) + + if len(ret) == 0 { + panic("no return value specified for GetVariant") + } + + var r0 domain.Variant + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int, domain.VariantName) (domain.Variant, error)); ok { + return returnFunc(ctx, attachmentID, pipelineVersion, name) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int, domain.VariantName) domain.Variant); ok { + r0 = returnFunc(ctx, attachmentID, pipelineVersion, name) + } else { + r0 = ret.Get(0).(domain.Variant) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, int, domain.VariantName) error); ok { + r1 = returnFunc(ctx, attachmentID, pipelineVersion, name) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockRepository_GetVariant_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetVariant' +type MockRepository_GetVariant_Call struct { + *mock.Call +} + +// GetVariant is a helper method to define mock.On call +// - ctx context.Context +// - attachmentID uuid.UUID +// - pipelineVersion int +// - name domain.VariantName +func (_e *MockRepository_Expecter) GetVariant(ctx any, attachmentID any, pipelineVersion any, name any) *MockRepository_GetVariant_Call { + return &MockRepository_GetVariant_Call{Call: _e.mock.On("GetVariant", ctx, attachmentID, pipelineVersion, name)} +} + +func (_c *MockRepository_GetVariant_Call) Run(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int, name domain.VariantName)) *MockRepository_GetVariant_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uuid.UUID + if args[1] != nil { + arg1 = args[1].(uuid.UUID) + } + var arg2 int + if args[2] != nil { + arg2 = args[2].(int) + } + var arg3 domain.VariantName + if args[3] != nil { + arg3 = args[3].(domain.VariantName) + } + run( + arg0, + arg1, + arg2, + arg3, + ) + }) + return _c +} + +func (_c *MockRepository_GetVariant_Call) Return(variant domain.Variant, err error) *MockRepository_GetVariant_Call { + _c.Call.Return(variant, err) + return _c +} + +func (_c *MockRepository_GetVariant_Call) RunAndReturn(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int, name domain.VariantName) (domain.Variant, error)) *MockRepository_GetVariant_Call { + _c.Call.Return(run) + return _c +} + // IsChatMember provides a mock function for the type MockRepository func (_mock *MockRepository) IsChatMember(ctx context.Context, chatID uuid.UUID, userID uuid.UUID) (bool, error) { ret := _mock.Called(ctx, chatID, userID) @@ -388,196 +729,48 @@ func (_c *MockRepository_IsChatMember_Call) RunAndReturn(run func(ctx context.Co return _c } -// ListKeys provides a mock function for the type MockRepository -func (_mock *MockRepository) ListKeys(ctx context.Context, limit int32, offset int32) ([]domain.AttachmentKey, error) { - ret := _mock.Called(ctx, limit, offset) +// ListVariants provides a mock function for the type MockRepository +func (_mock *MockRepository) ListVariants(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int) ([]domain.Variant, error) { + ret := _mock.Called(ctx, attachmentID, pipelineVersion) if len(ret) == 0 { - panic("no return value specified for ListKeys") + panic("no return value specified for ListVariants") } - var r0 []domain.AttachmentKey + var r0 []domain.Variant var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, int32, int32) ([]domain.AttachmentKey, error)); ok { - return returnFunc(ctx, limit, offset) + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int) ([]domain.Variant, error)); ok { + return returnFunc(ctx, attachmentID, pipelineVersion) } - if returnFunc, ok := ret.Get(0).(func(context.Context, int32, int32) []domain.AttachmentKey); ok { - r0 = returnFunc(ctx, limit, offset) + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int) []domain.Variant); ok { + r0 = returnFunc(ctx, attachmentID, pipelineVersion) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]domain.AttachmentKey) + r0 = ret.Get(0).([]domain.Variant) } } - if returnFunc, ok := ret.Get(1).(func(context.Context, int32, int32) error); ok { - r1 = returnFunc(ctx, limit, offset) + if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, int) error); ok { + r1 = returnFunc(ctx, attachmentID, pipelineVersion) } else { r1 = ret.Error(1) } return r0, r1 } -// MockRepository_ListKeys_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListKeys' -type MockRepository_ListKeys_Call struct { +// MockRepository_ListVariants_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListVariants' +type MockRepository_ListVariants_Call struct { *mock.Call } -// ListKeys is a helper method to define mock.On call +// ListVariants is a helper method to define mock.On call // - ctx context.Context -// - limit int32 -// - offset int32 -func (_e *MockRepository_Expecter) ListKeys(ctx any, limit any, offset any) *MockRepository_ListKeys_Call { - return &MockRepository_ListKeys_Call{Call: _e.mock.On("ListKeys", ctx, limit, offset)} +// - attachmentID uuid.UUID +// - pipelineVersion int +func (_e *MockRepository_Expecter) ListVariants(ctx any, attachmentID any, pipelineVersion any) *MockRepository_ListVariants_Call { + return &MockRepository_ListVariants_Call{Call: _e.mock.On("ListVariants", ctx, attachmentID, pipelineVersion)} } -func (_c *MockRepository_ListKeys_Call) Run(run func(ctx context.Context, limit int32, offset int32)) *MockRepository_ListKeys_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 int32 - if args[1] != nil { - arg1 = args[1].(int32) - } - var arg2 int32 - if args[2] != nil { - arg2 = args[2].(int32) - } - run( - arg0, - arg1, - arg2, - ) - }) - return _c -} - -func (_c *MockRepository_ListKeys_Call) Return(attachmentKeys []domain.AttachmentKey, err error) *MockRepository_ListKeys_Call { - _c.Call.Return(attachmentKeys, err) - return _c -} - -func (_c *MockRepository_ListKeys_Call) RunAndReturn(run func(ctx context.Context, limit int32, offset int32) ([]domain.AttachmentKey, error)) *MockRepository_ListKeys_Call { - _c.Call.Return(run) - return _c -} - -// ListStalePending provides a mock function for the type MockRepository -func (_mock *MockRepository) ListStalePending(ctx context.Context, createdBefore time.Time, limit int32) ([]domain.Attachment, error) { - ret := _mock.Called(ctx, createdBefore, limit) - - if len(ret) == 0 { - panic("no return value specified for ListStalePending") - } - - var r0 []domain.Attachment - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int32) ([]domain.Attachment, error)); ok { - return returnFunc(ctx, createdBefore, limit) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int32) []domain.Attachment); ok { - r0 = returnFunc(ctx, createdBefore, limit) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]domain.Attachment) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, time.Time, int32) error); ok { - r1 = returnFunc(ctx, createdBefore, limit) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// MockRepository_ListStalePending_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListStalePending' -type MockRepository_ListStalePending_Call struct { - *mock.Call -} - -// ListStalePending is a helper method to define mock.On call -// - ctx context.Context -// - createdBefore time.Time -// - limit int32 -func (_e *MockRepository_Expecter) ListStalePending(ctx any, createdBefore any, limit any) *MockRepository_ListStalePending_Call { - return &MockRepository_ListStalePending_Call{Call: _e.mock.On("ListStalePending", ctx, createdBefore, limit)} -} - -func (_c *MockRepository_ListStalePending_Call) Run(run func(ctx context.Context, createdBefore time.Time, limit int32)) *MockRepository_ListStalePending_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 time.Time - if args[1] != nil { - arg1 = args[1].(time.Time) - } - var arg2 int32 - if args[2] != nil { - arg2 = args[2].(int32) - } - run( - arg0, - arg1, - arg2, - ) - }) - return _c -} - -func (_c *MockRepository_ListStalePending_Call) Return(attachments []domain.Attachment, err error) *MockRepository_ListStalePending_Call { - _c.Call.Return(attachments, err) - return _c -} - -func (_c *MockRepository_ListStalePending_Call) RunAndReturn(run func(ctx context.Context, createdBefore time.Time, limit int32) ([]domain.Attachment, error)) *MockRepository_ListStalePending_Call { - _c.Call.Return(run) - return _c -} - -// ListStates provides a mock function for the type MockRepository -func (_mock *MockRepository) ListStates(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID) ([]domain.AttachmentState, error) { - ret := _mock.Called(ctx, chatID, ids) - - if len(ret) == 0 { - panic("no return value specified for ListStates") - } - - var r0 []domain.AttachmentState - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, []uuid.UUID) ([]domain.AttachmentState, error)); ok { - return returnFunc(ctx, chatID, ids) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, []uuid.UUID) []domain.AttachmentState); ok { - r0 = returnFunc(ctx, chatID, ids) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]domain.AttachmentState) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, []uuid.UUID) error); ok { - r1 = returnFunc(ctx, chatID, ids) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// MockRepository_ListStates_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListStates' -type MockRepository_ListStates_Call struct { - *mock.Call -} - -// ListStates is a helper method to define mock.On call -// - ctx context.Context -// - chatID uuid.UUID -// - ids []uuid.UUID -func (_e *MockRepository_Expecter) ListStates(ctx any, chatID any, ids any) *MockRepository_ListStates_Call { - return &MockRepository_ListStates_Call{Call: _e.mock.On("ListStates", ctx, chatID, ids)} -} - -func (_c *MockRepository_ListStates_Call) Run(run func(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID)) *MockRepository_ListStates_Call { +func (_c *MockRepository_ListVariants_Call) Run(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int)) *MockRepository_ListVariants_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -587,9 +780,9 @@ func (_c *MockRepository_ListStates_Call) Run(run func(ctx context.Context, chat if args[1] != nil { arg1 = args[1].(uuid.UUID) } - var arg2 []uuid.UUID + var arg2 int if args[2] != nil { - arg2 = args[2].([]uuid.UUID) + arg2 = args[2].(int) } run( arg0, @@ -600,84 +793,12 @@ func (_c *MockRepository_ListStates_Call) Run(run func(ctx context.Context, chat return _c } -func (_c *MockRepository_ListStates_Call) Return(attachmentStates []domain.AttachmentState, err error) *MockRepository_ListStates_Call { - _c.Call.Return(attachmentStates, err) +func (_c *MockRepository_ListVariants_Call) Return(variants []domain.Variant, err error) *MockRepository_ListVariants_Call { + _c.Call.Return(variants, err) return _c } -func (_c *MockRepository_ListStates_Call) RunAndReturn(run func(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID) ([]domain.AttachmentState, error)) *MockRepository_ListStates_Call { - _c.Call.Return(run) - return _c -} - -// MarkReady provides a mock function for the type MockRepository -func (_mock *MockRepository) MarkReady(ctx context.Context, id uuid.UUID, objectKey string) (domain.Attachment, error) { - ret := _mock.Called(ctx, id, objectKey) - - if len(ret) == 0 { - panic("no return value specified for MarkReady") - } - - var r0 domain.Attachment - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, string) (domain.Attachment, error)); ok { - return returnFunc(ctx, id, objectKey) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, string) domain.Attachment); ok { - r0 = returnFunc(ctx, id, objectKey) - } else { - r0 = ret.Get(0).(domain.Attachment) - } - if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, string) error); ok { - r1 = returnFunc(ctx, id, objectKey) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// MockRepository_MarkReady_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkReady' -type MockRepository_MarkReady_Call struct { - *mock.Call -} - -// MarkReady is a helper method to define mock.On call -// - ctx context.Context -// - id uuid.UUID -// - objectKey string -func (_e *MockRepository_Expecter) MarkReady(ctx any, id any, objectKey any) *MockRepository_MarkReady_Call { - return &MockRepository_MarkReady_Call{Call: _e.mock.On("MarkReady", ctx, id, objectKey)} -} - -func (_c *MockRepository_MarkReady_Call) Run(run func(ctx context.Context, id uuid.UUID, objectKey string)) *MockRepository_MarkReady_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 uuid.UUID - if args[1] != nil { - arg1 = args[1].(uuid.UUID) - } - var arg2 string - if args[2] != nil { - arg2 = args[2].(string) - } - run( - arg0, - arg1, - arg2, - ) - }) - return _c -} - -func (_c *MockRepository_MarkReady_Call) Return(attachment domain.Attachment, err error) *MockRepository_MarkReady_Call { - _c.Call.Return(attachment, err) - return _c -} - -func (_c *MockRepository_MarkReady_Call) RunAndReturn(run func(ctx context.Context, id uuid.UUID, objectKey string) (domain.Attachment, error)) *MockRepository_MarkReady_Call { +func (_c *MockRepository_ListVariants_Call) RunAndReturn(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int) ([]domain.Variant, error)) *MockRepository_ListVariants_Call { _c.Call.Return(run) return _c } diff --git a/internal/service/attachment/mocks/SweepRepository.go b/internal/service/attachment/mocks/SweepRepository.go new file mode 100644 index 00000000..e5e5e51a --- /dev/null +++ b/internal/service/attachment/mocks/SweepRepository.go @@ -0,0 +1,398 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package mocks + +import ( + "context" + "mtusi-messenger/internal/domain" + "time" + + "github.com/google/uuid" + mock "github.com/stretchr/testify/mock" +) + +// NewMockSweepRepository creates a new instance of MockSweepRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockSweepRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *MockSweepRepository { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + + mock := &MockSweepRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { + if helper, ok := t.(interface{ Helper() }); ok { + helper.Helper() + } + mock.AssertExpectations(t) + }) + + return mock +} + +// MockSweepRepository is an autogenerated mock type for the SweepRepository type +type MockSweepRepository struct { + mock.Mock +} + +type MockSweepRepository_Expecter struct { + mock *mock.Mock +} + +func (_m *MockSweepRepository) EXPECT() *MockSweepRepository_Expecter { + return &MockSweepRepository_Expecter{mock: &_m.Mock} +} + +// DeletePending provides a mock function for the type MockSweepRepository +func (_mock *MockSweepRepository) DeletePending(ctx context.Context, id uuid.UUID) error { + ret := _mock.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for DeletePending") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID) error); ok { + r0 = returnFunc(ctx, id) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockSweepRepository_DeletePending_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeletePending' +type MockSweepRepository_DeletePending_Call struct { + *mock.Call +} + +// DeletePending is a helper method to define mock.On call +// - ctx context.Context +// - id uuid.UUID +func (_e *MockSweepRepository_Expecter) DeletePending(ctx any, id any) *MockSweepRepository_DeletePending_Call { + return &MockSweepRepository_DeletePending_Call{Call: _e.mock.On("DeletePending", ctx, id)} +} + +func (_c *MockSweepRepository_DeletePending_Call) Run(run func(ctx context.Context, id uuid.UUID)) *MockSweepRepository_DeletePending_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uuid.UUID + if args[1] != nil { + arg1 = args[1].(uuid.UUID) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockSweepRepository_DeletePending_Call) Return(err error) *MockSweepRepository_DeletePending_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockSweepRepository_DeletePending_Call) RunAndReturn(run func(ctx context.Context, id uuid.UUID) error) *MockSweepRepository_DeletePending_Call { + _c.Call.Return(run) + return _c +} + +// ListPendingPresign provides a mock function for the type MockSweepRepository +func (_mock *MockSweepRepository) ListPendingPresign(ctx context.Context, createdBefore time.Time, limit int) ([]domain.Attachment, error) { + ret := _mock.Called(ctx, createdBefore, limit) + + if len(ret) == 0 { + panic("no return value specified for ListPendingPresign") + } + + var r0 []domain.Attachment + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int) ([]domain.Attachment, error)); ok { + return returnFunc(ctx, createdBefore, limit) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int) []domain.Attachment); ok { + r0 = returnFunc(ctx, createdBefore, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]domain.Attachment) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, time.Time, int) error); ok { + r1 = returnFunc(ctx, createdBefore, limit) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSweepRepository_ListPendingPresign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListPendingPresign' +type MockSweepRepository_ListPendingPresign_Call struct { + *mock.Call +} + +// ListPendingPresign is a helper method to define mock.On call +// - ctx context.Context +// - createdBefore time.Time +// - limit int +func (_e *MockSweepRepository_Expecter) ListPendingPresign(ctx any, createdBefore any, limit any) *MockSweepRepository_ListPendingPresign_Call { + return &MockSweepRepository_ListPendingPresign_Call{Call: _e.mock.On("ListPendingPresign", ctx, createdBefore, limit)} +} + +func (_c *MockSweepRepository_ListPendingPresign_Call) Run(run func(ctx context.Context, createdBefore time.Time, limit int)) *MockSweepRepository_ListPendingPresign_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 time.Time + if args[1] != nil { + arg1 = args[1].(time.Time) + } + var arg2 int + if args[2] != nil { + arg2 = args[2].(int) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockSweepRepository_ListPendingPresign_Call) Return(attachments []domain.Attachment, err error) *MockSweepRepository_ListPendingPresign_Call { + _c.Call.Return(attachments, err) + return _c +} + +func (_c *MockSweepRepository_ListPendingPresign_Call) RunAndReturn(run func(ctx context.Context, createdBefore time.Time, limit int) ([]domain.Attachment, error)) *MockSweepRepository_ListPendingPresign_Call { + _c.Call.Return(run) + return _c +} + +// ListReferencedKeys provides a mock function for the type MockSweepRepository +func (_mock *MockSweepRepository) ListReferencedKeys(ctx context.Context, afterKey string, limit int) ([]string, error) { + ret := _mock.Called(ctx, afterKey, limit) + + if len(ret) == 0 { + panic("no return value specified for ListReferencedKeys") + } + + var r0 []string + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, string, int) ([]string, error)); ok { + return returnFunc(ctx, afterKey, limit) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, string, int) []string); ok { + r0 = returnFunc(ctx, afterKey, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, string, int) error); ok { + r1 = returnFunc(ctx, afterKey, limit) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSweepRepository_ListReferencedKeys_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListReferencedKeys' +type MockSweepRepository_ListReferencedKeys_Call struct { + *mock.Call +} + +// ListReferencedKeys is a helper method to define mock.On call +// - ctx context.Context +// - afterKey string +// - limit int +func (_e *MockSweepRepository_Expecter) ListReferencedKeys(ctx any, afterKey any, limit any) *MockSweepRepository_ListReferencedKeys_Call { + return &MockSweepRepository_ListReferencedKeys_Call{Call: _e.mock.On("ListReferencedKeys", ctx, afterKey, limit)} +} + +func (_c *MockSweepRepository_ListReferencedKeys_Call) Run(run func(ctx context.Context, afterKey string, limit int)) *MockSweepRepository_ListReferencedKeys_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + var arg2 int + if args[2] != nil { + arg2 = args[2].(int) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockSweepRepository_ListReferencedKeys_Call) Return(strings []string, err error) *MockSweepRepository_ListReferencedKeys_Call { + _c.Call.Return(strings, err) + return _c +} + +func (_c *MockSweepRepository_ListReferencedKeys_Call) RunAndReturn(run func(ctx context.Context, afterKey string, limit int) ([]string, error)) *MockSweepRepository_ListReferencedKeys_Call { + _c.Call.Return(run) + return _c +} + +// ListStaleStaging provides a mock function for the type MockSweepRepository +func (_mock *MockSweepRepository) ListStaleStaging(ctx context.Context, readyBefore time.Time, failedBefore time.Time, limit int) ([]domain.StaleStaging, error) { + ret := _mock.Called(ctx, readyBefore, failedBefore, limit) + + if len(ret) == 0 { + panic("no return value specified for ListStaleStaging") + } + + var r0 []domain.StaleStaging + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, time.Time, int) ([]domain.StaleStaging, error)); ok { + return returnFunc(ctx, readyBefore, failedBefore, limit) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, time.Time, int) []domain.StaleStaging); ok { + r0 = returnFunc(ctx, readyBefore, failedBefore, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]domain.StaleStaging) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, time.Time, time.Time, int) error); ok { + r1 = returnFunc(ctx, readyBefore, failedBefore, limit) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSweepRepository_ListStaleStaging_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListStaleStaging' +type MockSweepRepository_ListStaleStaging_Call struct { + *mock.Call +} + +// ListStaleStaging is a helper method to define mock.On call +// - ctx context.Context +// - readyBefore time.Time +// - failedBefore time.Time +// - limit int +func (_e *MockSweepRepository_Expecter) ListStaleStaging(ctx any, readyBefore any, failedBefore any, limit any) *MockSweepRepository_ListStaleStaging_Call { + return &MockSweepRepository_ListStaleStaging_Call{Call: _e.mock.On("ListStaleStaging", ctx, readyBefore, failedBefore, limit)} +} + +func (_c *MockSweepRepository_ListStaleStaging_Call) Run(run func(ctx context.Context, readyBefore time.Time, failedBefore time.Time, limit int)) *MockSweepRepository_ListStaleStaging_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 time.Time + if args[1] != nil { + arg1 = args[1].(time.Time) + } + var arg2 time.Time + if args[2] != nil { + arg2 = args[2].(time.Time) + } + var arg3 int + if args[3] != nil { + arg3 = args[3].(int) + } + run( + arg0, + arg1, + arg2, + arg3, + ) + }) + return _c +} + +func (_c *MockSweepRepository_ListStaleStaging_Call) Return(staleStagings []domain.StaleStaging, err error) *MockSweepRepository_ListStaleStaging_Call { + _c.Call.Return(staleStagings, err) + return _c +} + +func (_c *MockSweepRepository_ListStaleStaging_Call) RunAndReturn(run func(ctx context.Context, readyBefore time.Time, failedBefore time.Time, limit int) ([]domain.StaleStaging, error)) *MockSweepRepository_ListStaleStaging_Call { + _c.Call.Return(run) + return _c +} + +// MarkStagingDeleted provides a mock function for the type MockSweepRepository +func (_mock *MockSweepRepository) MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error { + ret := _mock.Called(ctx, attachmentID, stagingKey) + + if len(ret) == 0 { + panic("no return value specified for MarkStagingDeleted") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, string) error); ok { + r0 = returnFunc(ctx, attachmentID, stagingKey) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockSweepRepository_MarkStagingDeleted_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkStagingDeleted' +type MockSweepRepository_MarkStagingDeleted_Call struct { + *mock.Call +} + +// MarkStagingDeleted is a helper method to define mock.On call +// - ctx context.Context +// - attachmentID uuid.UUID +// - stagingKey string +func (_e *MockSweepRepository_Expecter) MarkStagingDeleted(ctx any, attachmentID any, stagingKey any) *MockSweepRepository_MarkStagingDeleted_Call { + return &MockSweepRepository_MarkStagingDeleted_Call{Call: _e.mock.On("MarkStagingDeleted", ctx, attachmentID, stagingKey)} +} + +func (_c *MockSweepRepository_MarkStagingDeleted_Call) Run(run func(ctx context.Context, attachmentID uuid.UUID, stagingKey string)) *MockSweepRepository_MarkStagingDeleted_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uuid.UUID + if args[1] != nil { + arg1 = args[1].(uuid.UUID) + } + var arg2 string + if args[2] != nil { + arg2 = args[2].(string) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockSweepRepository_MarkStagingDeleted_Call) Return(err error) *MockSweepRepository_MarkStagingDeleted_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockSweepRepository_MarkStagingDeleted_Call) RunAndReturn(run func(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error) *MockSweepRepository_MarkStagingDeleted_Call { + _c.Call.Return(run) + return _c +} diff --git a/internal/service/attachment/service.go b/internal/service/attachment/service.go index fd25c682..a4d39b4d 100644 --- a/internal/service/attachment/service.go +++ b/internal/service/attachment/service.go @@ -17,6 +17,7 @@ import ( "unicode/utf8" "github.com/google/uuid" + "golang.org/x/sync/semaphore" "mtusi-messenger/internal/domain" "mtusi-messenger/internal/imagecheck" @@ -27,32 +28,61 @@ const ( contentTypePNG = "image/png" contentTypeJPEG = "image/jpeg" - objectPrefixAttachments = "attachments/" - objectPrefixStaging = "staging/" - objectPrefixPresign = "staging/presign/" + objectPrefixAttachments = domain.AttachmentsPrefix + objectPrefixStaging = domain.StagingUploadPrefix + objectPrefixPresign = domain.StagingPresignPrefix ) var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) // Repository is the persistence port. It is consumer-side so the service can be -// tested without PostgreSQL. +// tested without PostgreSQL. Job/outbox writes are atomic with the attachment +// row inside CreateQueued/CompletePresign. type Repository interface { IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) - Create(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error) - MarkReady(ctx context.Context, id uuid.UUID, objectKey string) (domain.Attachment, error) - DeletePending(ctx context.Context, id uuid.UUID) error GetForMember(ctx context.Context, userID, id uuid.UUID) (domain.Attachment, error) GetForOwner(ctx context.Context, ownerID, id uuid.UUID) (domain.Attachment, error) - ListStates(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID) ([]domain.AttachmentState, error) - ListStalePending(ctx context.Context, createdBefore time.Time, limit int32) ([]domain.Attachment, error) - ListKeys(ctx context.Context, limit, offset int32) ([]domain.AttachmentKey, error) + ListVariants(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int) ([]domain.Variant, error) + GetVariant( + ctx context.Context, + attachmentID uuid.UUID, + pipelineVersion int, + name domain.VariantName, + ) (domain.Variant, error) + CreateQueued( + ctx context.Context, + draft domain.AttachmentDraft, + job domain.Job, + limits domain.AdmissionLimits, + ) (domain.Attachment, error) + CreatePresign( + ctx context.Context, + draft domain.AttachmentDraft, + limits domain.AdmissionLimits, + ) (domain.Attachment, error) + CompletePresign( + ctx context.Context, + ownerID, attachmentID uuid.UUID, + verifiedKey, finalKey string, + pipelineVersion, maxAttempts int, + jobID uuid.UUID, + ) (domain.Attachment, error) + Admission(ctx context.Context, ownerID uuid.UUID) (userPending, totalPending int, stagedBytes int64, err error) + DeletePending(ctx context.Context, id uuid.UUID) error } -// Config carries the attachment limits. They are validated by config.Load. +// Config carries attachment limits and the shared job contract knobs. type Config struct { MaxUploadBytes int64 MaxImagePixels int64 PresignTTL time.Duration + + PipelineVersion int + MaxAttempts int + MaxPendingPerUser int + MaxPending int + MaxStagedBytes int64 + MaxConcurrent int } type Service struct { @@ -60,6 +90,7 @@ type Service struct { store objectstore.Store config Config log *slog.Logger + upload *semaphore.Weighted } func NewService(repo Repository, store objectstore.Store, config Config, log *slog.Logger) *Service { @@ -67,7 +98,12 @@ func NewService(repo Repository, store objectstore.Store, config Config, log *sl log = slog.Default() } - return &Service{repo: repo, store: store, config: config, log: log} + service := &Service{repo: repo, store: store, config: config, log: log} + if config.MaxConcurrent > 0 { + service.upload = semaphore.NewWeighted(int64(config.MaxConcurrent)) + } + + return service } // UploadInput is one raw-body upload. DeclaredContentType comes from the HTTP @@ -80,7 +116,16 @@ type UploadInput struct { Body io.Reader } +// Upload durably stages the original, records the job intent and returns the +// queued snapshot. It never waits for the broker or the worker. func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachment, error) { + if s.upload != nil { + if err := s.upload.Acquire(ctx, 1); err != nil { + return domain.Attachment{}, unavailable("acquire upload slot", err) + } + defer s.upload.Release(1) + } + if err := ValidateFilename(input.Filename); err != nil { return domain.Attachment{}, err } @@ -94,6 +139,17 @@ func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachm return domain.Attachment{}, fmt.Errorf("%w: chat is not visible", domain.ErrNotFound) } + userPending, totalPending, stagedBytes, err := s.repo.Admission(ctx, input.OwnerID) + if err != nil { + return domain.Attachment{}, unavailable("check admission", err) + } + + if (s.config.MaxPendingPerUser > 0 && userPending >= s.config.MaxPendingPerUser) || + (s.config.MaxPending > 0 && totalPending >= s.config.MaxPending) || + (s.config.MaxStagedBytes > 0 && stagedBytes >= s.config.MaxStagedBytes) { + return domain.Attachment{}, fmt.Errorf("%w: too many unfinished uploads", domain.ErrRateLimited) + } + file, size, sum, err := s.spool(input.Body) if err != nil { return domain.Attachment{}, err @@ -123,79 +179,161 @@ func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachm return domain.Attachment{}, fmt.Errorf("generate attachment id: %w", err) } - objectKey := objectPrefixAttachments + input.ChatID.String() + "/" + id.String() - - if _, err := s.repo.Create(ctx, domain.AttachmentDraft{ - ID: id, - ChatID: input.ChatID, - OwnerID: input.OwnerID, - ObjectKey: objectKey, - Filename: input.Filename, - ContentType: info.ContentType, - SizeBytes: size, - SHA256: sum, - }); err != nil { - return domain.Attachment{}, err + jobID, err := uuid.NewV7() + if err != nil { + return domain.Attachment{}, fmt.Errorf("generate job id: %w", err) } + stagingKey := domain.StagingUploadKey(input.ChatID, id) + finalKey := domain.OriginalObjectKey(input.ChatID, id) + if _, err := file.Seek(0, io.SeekStart); err != nil { return domain.Attachment{}, fmt.Errorf("rewind upload: %w", err) } - if err := s.store.Put(ctx, objectKey, file, size, info.ContentType, map[string]string{"sha256": sum}); err != nil { - s.cleanupPending(ctx, id, "") - - return domain.Attachment{}, unavailable("store attachment", err) + if err := s.store.Put(ctx, stagingKey, file, size, info.ContentType, map[string]string{"sha256": sum}); err != nil { + return domain.Attachment{}, unavailable("store staging object", err) } - ready, err := s.repo.MarkReady(ctx, id, objectKey) + if err := s.verifyStaging(ctx, stagingKey, size, sum); err != nil { + s.deleteObject(ctx, id, stagingKey) + + return domain.Attachment{}, unavailable("verify staging object", err) + } + + job := domain.Job{ + ID: jobID, + AttachmentID: id, + PipelineVersion: s.config.PipelineVersion, + SourceKey: stagingKey, + Attempt: 1, + MaxAttempts: s.config.MaxAttempts, + CreatedAt: time.Now().UTC(), + } + + limits := s.admissionLimits() + + created, err := s.repo.CreateQueued(ctx, domain.AttachmentDraft{ + ID: id, + ChatID: input.ChatID, + OwnerID: input.OwnerID, + ObjectKey: finalKey, + StagingKey: stagingKey, + Filename: input.Filename, + ContentType: info.ContentType, + SizeBytes: size, + SHA256: sum, + Status: domain.AttachmentQueued, + PipelineVersion: s.config.PipelineVersion, + }, job, limits) if err != nil { - // The object and the pending row remain; the sweeper either finalises - // the row or removes both after the grace period. The client must not - // receive 201 until metadata is committed. - return domain.Attachment{}, unavailable("commit attachment", err) + if errors.Is(err, domain.ErrRateLimited) { + s.deleteObject(ctx, id, stagingKey) + + return domain.Attachment{}, err + } + + // An ambiguous COMMIT may have persisted the intent; leave the staging + // object for the sweeper instead of destroying a queued job's source. + return domain.Attachment{}, unavailable("commit attachment intent", err) } - return ready, nil + created.Variants = []domain.Variant{} + + return created, nil } +// Get returns metadata for any public status. Variants are exposed only for the +// currently published pipeline version and only when ready. func (s *Service) Get(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, error) { attachment, err := s.repo.GetForMember(ctx, callerID, id) if err != nil { return domain.Attachment{}, err } - // Contract 1.4.0 declares Attachment.status as the single value "ready". - // Pending/failed rows therefore cannot be represented in a 200 response; - // they are reported as 409 until lab5 extends the enum. - if attachment.Status != domain.AttachmentReady { + if !attachment.Status.Public() { return domain.Attachment{}, fmt.Errorf("%w: attachment is %s", domain.ErrConflict, attachment.Status) } - return attachment, nil + return s.withVariants(ctx, attachment) } -// Open returns metadata and the original stream. Only ready attachments have -// bytes; pending/failed return 409 on the content endpoint. -func (s *Service) Open( +// OpenVariant returns metadata plus the selected content stream. Only a ready +// attachment has bytes. +func (s *Service) OpenVariant( ctx context.Context, callerID, id uuid.UUID, -) (domain.Attachment, io.ReadCloser, error) { + name domain.VariantName, +) (domain.Attachment, domain.Variant, io.ReadCloser, error) { attachment, err := s.repo.GetForMember(ctx, callerID, id) if err != nil { - return domain.Attachment{}, nil, err + return domain.Attachment{}, domain.Variant{}, nil, err } + if !attachment.Status.Public() || attachment.Status != domain.AttachmentReady { + return domain.Attachment{}, domain.Variant{}, nil, fmt.Errorf( + "%w: attachment is %s", domain.ErrConflict, attachment.Status, + ) + } + + if name == "" || name == domain.VariantOriginal { + body, _, err := s.store.Open(ctx, attachment.ObjectKey) + if err != nil { + return domain.Attachment{}, domain.Variant{}, nil, err + } + + return attachment, domain.Variant{ + Name: domain.VariantOriginal, + ContentType: attachment.ContentType, + SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, + ObjectKey: attachment.ObjectKey, + }, body, nil + } + + if !name.Valid() { + return domain.Attachment{}, domain.Variant{}, nil, fmt.Errorf( + "%w: variant %q", domain.ErrValidation, name, + ) + } + + variant, err := s.repo.GetVariant(ctx, attachment.ID, attachment.PipelineVersion, name) + if err != nil { + return domain.Attachment{}, domain.Variant{}, nil, err + } + + body, _, err := s.store.Open(ctx, variant.ObjectKey) + if err != nil { + return domain.Attachment{}, domain.Variant{}, nil, err + } + + return attachment, variant, body, nil +} + +// Open is kept for the legacy service port used by tests. +func (s *Service) Open(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error) { + attachment, _, body, err := s.OpenVariant(ctx, callerID, id, domain.VariantOriginal) + + return attachment, body, err +} + +func (s *Service) withVariants(ctx context.Context, attachment domain.Attachment) (domain.Attachment, error) { + attachment.Variants = []domain.Variant{} + if attachment.Status != domain.AttachmentReady { - return domain.Attachment{}, nil, fmt.Errorf("%w: attachment is %s", domain.ErrConflict, attachment.Status) + return attachment, nil } - body, _, err := s.store.Open(ctx, attachment.ObjectKey) + variants, err := s.repo.ListVariants(ctx, attachment.ID, attachment.PipelineVersion) if err != nil { - return domain.Attachment{}, nil, err + return domain.Attachment{}, err } - return attachment, body, nil + if variants != nil { + attachment.Variants = variants + } + + return attachment, nil } func (s *Service) Presign(ctx context.Context, request domain.PresignRequest) (domain.PresignResult, error) { @@ -235,16 +373,19 @@ func (s *Service) Presign(ctx context.Context, request domain.PresignRequest) (d stagingKey := objectPrefixPresign + request.ChatID.String() + "/" + id.String() - if _, err := s.repo.Create(ctx, domain.AttachmentDraft{ - ID: id, - ChatID: request.ChatID, - OwnerID: request.OwnerID, - ObjectKey: stagingKey, - Filename: request.Filename, - ContentType: request.ContentType, - SizeBytes: request.SizeBytes, - SHA256: request.SHA256, - }); err != nil { + if _, err := s.repo.CreatePresign(ctx, domain.AttachmentDraft{ + ID: id, + ChatID: request.ChatID, + OwnerID: request.OwnerID, + ObjectKey: stagingKey, + StagingKey: stagingKey, + Filename: request.Filename, + ContentType: request.ContentType, + SizeBytes: request.SizeBytes, + SHA256: request.SHA256, + Status: domain.AttachmentPending, + PipelineVersion: s.config.PipelineVersion, + }, s.admissionLimits()); err != nil { return domain.PresignResult{}, err } @@ -264,6 +405,10 @@ func (s *Service) Presign(ctx context.Context, request domain.PresignRequest) (d }, nil } +// Complete verifies the presigned object, copies the verified bytes to a +// server-owned staging key, and enqueues the same async pipeline as the proxy +// upload. The client's staging key remains writable until the verified copy is +// durable. func (s *Service) Complete( ctx context.Context, ownerID, id uuid.UUID, @@ -279,12 +424,12 @@ func (s *Service) Complete( } switch attachment.Status { - case domain.AttachmentReady: + case domain.AttachmentReady, domain.AttachmentQueued, domain.AttachmentProcessing: if attachment.SHA256 != requestSHA { return domain.Attachment{}, fmt.Errorf("%w: attachment already completed with another checksum", domain.ErrConflict) } - return attachment, nil + return s.withVariants(ctx, attachment) case domain.AttachmentFailed: return domain.Attachment{}, fmt.Errorf("%w: attachment is failed", domain.ErrConflict) case domain.AttachmentPending: @@ -319,18 +464,31 @@ func (s *Service) Complete( return domain.Attachment{}, err } - hasher := sha256.New() - info, validateErr := imagecheck.Validate(io.TeeReader(body, hasher), s.config.MaxImagePixels) - closeErr := body.Close() + file, size, sum, err := s.spool(body) + _ = body.Close() - if validateErr != nil { - s.cleanupPending(ctx, id, attachment.ObjectKey) - - return domain.Attachment{}, mapImageError(validateErr) + if err != nil { + return domain.Attachment{}, err } - if closeErr != nil { - return domain.Attachment{}, unavailable("read uploaded object", closeErr) + defer func() { _ = file.Close() }() + defer func() { _ = os.Remove(file.Name()) }() + + if size != attachment.SizeBytes { + s.cleanupPending(ctx, id, attachment.ObjectKey) + + return domain.Attachment{}, fmt.Errorf("%w: uploaded object size changed", domain.ErrValidation) + } + + if _, err := file.Seek(0, io.SeekStart); err != nil { + return domain.Attachment{}, fmt.Errorf("rewind uploaded object: %w", err) + } + + info, err := imagecheck.Validate(file, s.config.MaxImagePixels) + if err != nil { + s.cleanupPending(ctx, id, attachment.ObjectKey) + + return domain.Attachment{}, mapImageError(err) } if info.ContentType != attachment.ContentType { @@ -342,7 +500,6 @@ func (s *Service) Complete( ) } - sum := hex.EncodeToString(hasher.Sum(nil)) if sum != attachment.SHA256 { s.cleanupPending(ctx, id, attachment.ObjectKey) @@ -350,43 +507,94 @@ func (s *Service) Complete( } if requestSHA != attachment.SHA256 { - // The object is valid; only the completion request is inconsistent. - // Keep the pending row and staging object so a retry can succeed. return domain.Attachment{}, fmt.Errorf("%w: completion checksum mismatch", domain.ErrConflict) } - finalKey := objectPrefixAttachments + attachment.ChatID.String() + "/" + id.String() + verifiedKey := domain.StagingVerifiedKey(attachment.ChatID, attachment.ID) + finalKey := domain.OriginalObjectKey(attachment.ChatID, attachment.ID) - if err := s.store.Copy(ctx, attachment.ObjectKey, finalKey, attachment.ContentType, map[string]string{"sha256": sum}); err != nil { - return domain.Attachment{}, unavailable("publish attachment", err) + if _, err := file.Seek(0, io.SeekStart); err != nil { + return domain.Attachment{}, fmt.Errorf("rewind verified object: %w", err) } - ready, err := s.repo.MarkReady(ctx, id, finalKey) + if err := s.store.Put(ctx, verifiedKey, file, size, info.ContentType, map[string]string{"sha256": sum}); err != nil { + return domain.Attachment{}, unavailable("store verified object", err) + } + + if err := s.verifyStaging(ctx, verifiedKey, size, sum); err != nil { + s.deleteObject(ctx, id, verifiedKey) + + return domain.Attachment{}, unavailable("verify staging object", err) + } + + jobID, err := uuid.NewV7() if err != nil { - // A concurrent completion may have won. Re-read instead of reporting a - // spurious failure. - current, getErr := s.repo.GetForOwner(ctx, ownerID, id) - if getErr == nil && current.Status == domain.AttachmentReady && current.SHA256 == requestSHA { - return current, nil + s.deleteObject(ctx, id, verifiedKey) + + return domain.Attachment{}, fmt.Errorf("generate job id: %w", err) + } + + queued, err := s.repo.CompletePresign( + ctx, ownerID, id, verifiedKey, finalKey, + s.config.PipelineVersion, s.config.MaxAttempts, jobID, + ) + if err != nil { + // The verified key is shared by concurrent completions of this + // attachment. Never delete it here: a winning request may already have + // queued a job pointing at it. The orphan sweeper owns abandoned keys. + if errors.Is(err, domain.ErrConflict) { + current, getErr := s.repo.GetForOwner(ctx, ownerID, id) + if getErr == nil && current.SHA256 == requestSHA && current.Status.Public() { + return s.withVariants(ctx, current) + } + + return domain.Attachment{}, fmt.Errorf("%w: attachment is no longer pending", domain.ErrConflict) } - return domain.Attachment{}, unavailable("commit attachment", err) + return domain.Attachment{}, unavailable("commit presigned attachment", err) } - if err := s.store.Delete(ctx, attachment.ObjectKey); err != nil { - // The final object is already committed; a stale staging object is an - // orphan and the sweeper removes it after the grace period. - s.log.WarnContext(ctx, "staging object cleanup failed", + if err := s.store.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, domain.ErrNotFound) { + s.log.WarnContext(ctx, "presign staging cleanup failed", slog.String("attachment_id", id.String()), slog.Any("error", err), ) } - return ready, nil + queued.Variants = []domain.Variant{} + + return queued, nil +} + +// verifyStaging checks the freshly written object; a mismatch is a broken +// staging handoff and must never become a 202. +func (s *Service) admissionLimits() domain.AdmissionLimits { + return domain.AdmissionLimits{ + PerUser: s.config.MaxPendingPerUser, + Total: s.config.MaxPending, + Bytes: s.config.MaxStagedBytes, + } +} + +func (s *Service) verifyStaging(ctx context.Context, key string, size int64, sum string) error { + head, err := s.store.Head(ctx, key) + if err != nil { + return fmt.Errorf("head staging object: %w", err) + } + + if head.Size != size { + return fmt.Errorf("staging size %d, want %d", head.Size, size) + } + + if sha := head.Metadata["sha256"]; sha != "" && sha != sum { + return errors.New("staging checksum metadata does not match") + } + + return nil } func (s *Service) spool(body io.Reader) (*os.File, int64, string, error) { - file, err := os.CreateTemp("", "lab4-upload-*") + file, err := os.CreateTemp("", "lab5-upload-*") if err != nil { return nil, 0, "", fmt.Errorf("create upload spool: %w", err) } @@ -427,16 +635,22 @@ func (s *Service) spool(body io.Reader) (*os.File, int64, string, error) { return file, size, hex.EncodeToString(hasher.Sum(nil)), nil } -func (s *Service) cleanupPending(ctx context.Context, id uuid.UUID, objectKey string) { - if objectKey != "" { - if err := s.store.Delete(ctx, objectKey); err != nil && !errors.Is(err, domain.ErrNotFound) { - s.log.WarnContext(ctx, "object cleanup failed", - slog.String("attachment_id", id.String()), - slog.Any("error", err), - ) - } +func (s *Service) deleteObject(ctx context.Context, id uuid.UUID, key string) { + if key == "" { + return } + if err := s.store.Delete(ctx, key); err != nil && !errors.Is(err, domain.ErrNotFound) { + s.log.WarnContext(ctx, "object cleanup failed", + slog.String("attachment_id", id.String()), + slog.Any("error", err), + ) + } +} + +func (s *Service) cleanupPending(ctx context.Context, id uuid.UUID, objectKey string) { + s.deleteObject(ctx, id, objectKey) + if err := s.repo.DeletePending(ctx, id); err != nil && !errors.Is(err, domain.ErrNotFound) { s.log.WarnContext(ctx, "pending attachment cleanup failed", slog.String("attachment_id", id.String()), diff --git a/internal/service/attachment/service_test.go b/internal/service/attachment/service_test.go index 1e9839b9..9c065fd5 100644 --- a/internal/service/attachment/service_test.go +++ b/internal/service/attachment/service_test.go @@ -71,6 +71,8 @@ func (f *fakeStore) Open(_ context.Context, key string) (io.ReadCloser, objectst return io.NopCloser(bytes.NewReader(object.data)), f.object(key, object), nil } +func (f *fakeStore) Ping(context.Context) error { return nil } + func (f *fakeStore) Head(_ context.Context, key string) (objectstore.Object, error) { object, ok := f.objects[key] if !ok { @@ -144,9 +146,11 @@ func pngBytes(t *testing.T, width, height int) []byte { func serviceConfig() attachment.Config { return attachment.Config{ - MaxUploadBytes: 1 << 20, - MaxImagePixels: 1_000_000, - PresignTTL: 5 * time.Minute, + MaxUploadBytes: 1 << 20, + MaxImagePixels: 1_000_000, + PresignTTL: 5 * time.Minute, + PipelineVersion: 1, + MaxAttempts: 5, } } @@ -162,7 +166,7 @@ func TestValidateFilename(t *testing.T) { } } -func TestUploadStoresOriginal(t *testing.T) { +func TestUploadStagesAndQueues(t *testing.T) { repo := mocks.NewMockRepository(t) store := newFakeStore() service := attachment.NewService(repo, store, serviceConfig(), slog.Default()) @@ -171,16 +175,25 @@ func TestUploadStoresOriginal(t *testing.T) { ownerID := uuid.New() content := pngBytes(t, 16, 16) sum := sha256.Sum256(content) + digest := hex.EncodeToString(sum[:]) repo.EXPECT().IsChatMember(mock.Anything, chatID, ownerID).Return(true, nil).Once() - repo.EXPECT().Create(mock.Anything, mock.MatchedBy(func(draft domain.AttachmentDraft) bool { + repo.EXPECT().Admission(mock.Anything, ownerID).Return(0, 0, int64(0), nil).Once() + repo.EXPECT().CreateQueued(mock.Anything, mock.MatchedBy(func(draft domain.AttachmentDraft) bool { return draft.ChatID == chatID && draft.OwnerID == ownerID && draft.ContentType == "image/png" && draft.SizeBytes == int64(len(content)) && - draft.SHA256 == hex.EncodeToString(sum[:]) && strings.HasPrefix(draft.ObjectKey, "attachments/") - })).Return(domain.Attachment{Status: domain.AttachmentPending}, nil).Once() - repo.EXPECT().MarkReady(mock.Anything, mock.Anything, mock.Anything). - Return(domain.Attachment{Status: domain.AttachmentReady, ContentType: "image/png", SizeBytes: int64(len(content))}, nil). - Once() + draft.SHA256 == digest && draft.Status == domain.AttachmentQueued && + draft.PipelineVersion == 1 && + strings.HasPrefix(draft.ObjectKey, "attachments/") && + strings.HasPrefix(draft.StagingKey, "staging/uploads/") + }), mock.MatchedBy(func(job domain.Job) bool { + return job.PipelineVersion == 1 && job.Attempt == 1 && + job.MaxAttempts == 5 && job.SourceKey != "" + }), domain.AdmissionLimits{}).Return(domain.Attachment{ + ID: uuid.New(), ChatID: chatID, OwnerID: ownerID, + ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest, + Status: domain.AttachmentQueued, PipelineVersion: 1, + }, nil).Once() result, err := service.Upload(t.Context(), attachment.UploadInput{ ChatID: chatID, @@ -190,28 +203,28 @@ func TestUploadStoresOriginal(t *testing.T) { Body: bytes.NewReader(content), }) require.NoError(t, err) - require.Equal(t, domain.AttachmentReady, result.Status) + require.Equal(t, domain.AttachmentQueued, result.Status) require.Len(t, store.objects, 1) + for key := range store.objects { + require.Contains(t, key, "staging/uploads/") + } } func TestUploadRejectsTooLargeBody(t *testing.T) { repo := mocks.NewMockRepository(t) service := attachment.NewService(repo, newFakeStore(), attachment.Config{ - MaxUploadBytes: 8, - MaxImagePixels: 1_000_000, - PresignTTL: time.Minute, + MaxUploadBytes: 8, MaxImagePixels: 1_000_000, PresignTTL: time.Minute, + PipelineVersion: 1, MaxAttempts: 5, }, slog.Default()) repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once() + repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(0, 0, int64(0), nil).Once() _, err := service.Upload(t.Context(), attachment.UploadInput{ - ChatID: uuid.New(), - OwnerID: uuid.New(), - Filename: "photo.png", + ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png", DeclaredContentType: "image/png", Body: bytes.NewReader(bytes.Repeat([]byte("x"), 9)), }) - require.ErrorIs(t, err, domain.ErrTooLarge) } @@ -220,19 +233,63 @@ func TestUploadRejectsMIMEMismatch(t *testing.T) { service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default()) repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once() + repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(0, 0, int64(0), nil).Once() _, err := service.Upload(t.Context(), attachment.UploadInput{ - ChatID: uuid.New(), - OwnerID: uuid.New(), - Filename: "photo.jpg", + ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.jpg", DeclaredContentType: "image/jpeg", Body: bytes.NewReader(pngBytes(t, 8, 8)), }) - require.ErrorIs(t, err, domain.ErrUnsupportedMedia) } -func TestGetRejectsNonReadyAttachment(t *testing.T) { +func TestUploadAdmissionRejectsFlood(t *testing.T) { + repo := mocks.NewMockRepository(t) + service := attachment.NewService(repo, newFakeStore(), attachment.Config{ + MaxUploadBytes: 1 << 20, MaxImagePixels: 1_000_000, PresignTTL: time.Minute, + PipelineVersion: 1, MaxAttempts: 5, MaxPendingPerUser: 1, + }, slog.Default()) + + repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once() + repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(1, 1, int64(0), nil).Once() + + _, err := service.Upload(t.Context(), attachment.UploadInput{ + ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png", + DeclaredContentType: "image/png", Body: bytes.NewReader(pngBytes(t, 8, 8)), + }) + require.ErrorIs(t, err, domain.ErrRateLimited) +} + +func TestGetReturnsQueuedSnapshotWithoutVariants(t *testing.T) { + repo := mocks.NewMockRepository(t) + service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default()) + + repo.EXPECT().GetForMember(mock.Anything, mock.Anything, mock.Anything). + Return(domain.Attachment{Status: domain.AttachmentQueued, PipelineVersion: 1}, nil).Once() + + got, err := service.Get(t.Context(), uuid.New(), uuid.New()) + require.NoError(t, err) + require.Equal(t, domain.AttachmentQueued, got.Status) + require.Empty(t, got.Variants) +} + +func TestGetReadyLoadsCurrentVariants(t *testing.T) { + repo := mocks.NewMockRepository(t) + service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default()) + + attachmentID := uuid.New() + variant := domain.Variant{Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 10, Width: 256, Height: 160, SHA256: strings.Repeat("a", 64)} + + repo.EXPECT().GetForMember(mock.Anything, mock.Anything, attachmentID). + Return(domain.Attachment{ID: attachmentID, Status: domain.AttachmentReady, PipelineVersion: 2}, nil).Once() + repo.EXPECT().ListVariants(mock.Anything, attachmentID, 2).Return([]domain.Variant{variant}, nil).Once() + + got, err := service.Get(t.Context(), uuid.New(), attachmentID) + require.NoError(t, err) + require.Equal(t, []domain.Variant{variant}, got.Variants) +} + +func TestGetRejectsInternalPending(t *testing.T) { repo := mocks.NewMockRepository(t) service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default()) @@ -243,7 +300,18 @@ func TestGetRejectsNonReadyAttachment(t *testing.T) { require.ErrorIs(t, err, domain.ErrConflict) } -func TestCompletePublishesPresignedObject(t *testing.T) { +func TestOpenVariantRejectsQueued(t *testing.T) { + repo := mocks.NewMockRepository(t) + service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default()) + + repo.EXPECT().GetForMember(mock.Anything, mock.Anything, mock.Anything). + Return(domain.Attachment{Status: domain.AttachmentQueued}, nil).Once() + + _, _, _, err := service.OpenVariant(t.Context(), uuid.New(), uuid.New(), domain.VariantOriginal) + require.ErrorIs(t, err, domain.ErrConflict) +} + +func TestCompleteQueuesPresignedObject(t *testing.T) { repo := mocks.NewMockRepository(t) store := newFakeStore() service := attachment.NewService(repo, store, serviceConfig(), slog.Default()) @@ -257,33 +325,119 @@ func TestCompletePublishesPresignedObject(t *testing.T) { stagingKey := "staging/presign/" + chatID.String() + "/" + id.String() store.objects[stagingKey] = storedObject{ - data: content, - contentType: "image/png", - metadata: map[string]string{"sha256": digest}, - lastModified: time.Now().UTC(), + data: content, contentType: "image/png", + metadata: map[string]string{"sha256": digest}, lastModified: time.Now().UTC(), } pending := domain.Attachment{ ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey, - ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest, - Status: domain.AttachmentPending, + StagingKey: stagingKey, ContentType: "image/png", + SizeBytes: int64(len(content)), SHA256: digest, + Status: domain.AttachmentPending, PipelineVersion: 1, } repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once() - repo.EXPECT().MarkReady(mock.Anything, id, mock.MatchedBy(func(key string) bool { - return strings.HasPrefix(key, "attachments/") - })).Return(domain.Attachment{ + repo.EXPECT().CompletePresign( + mock.Anything, ownerID, id, + mock.MatchedBy(func(key string) bool { return strings.HasPrefix(key, "staging/verified/") }), + mock.MatchedBy(func(key string) bool { return strings.HasPrefix(key, "attachments/") }), + 1, 5, mock.Anything, + ).Return(domain.Attachment{ ID: id, ChatID: chatID, OwnerID: ownerID, ContentType: "image/png", - SizeBytes: int64(len(content)), SHA256: digest, Status: domain.AttachmentReady, + SizeBytes: int64(len(content)), SHA256: digest, + Status: domain.AttachmentQueued, PipelineVersion: 1, }, nil).Once() completed, err := service.Complete(t.Context(), ownerID, id, digest) require.NoError(t, err) - require.Equal(t, domain.AttachmentReady, completed.Status) + require.Equal(t, domain.AttachmentQueued, completed.Status) require.NotContains(t, store.objects, stagingKey) require.Len(t, store.objects, 1) } +func TestCompleteConcurrentConflictKeepsVerifiedSource(t *testing.T) { + repo := mocks.NewMockRepository(t) + store := newFakeStore() + service := attachment.NewService(repo, store, serviceConfig(), slog.Default()) + + id := uuid.New() + chatID := uuid.New() + ownerID := uuid.New() + content := pngBytes(t, 10, 10) + sum := sha256.Sum256(content) + digest := hex.EncodeToString(sum[:]) + stagingKey := "staging/presign/" + chatID.String() + "/" + id.String() + + store.objects[stagingKey] = storedObject{ + data: content, contentType: "image/png", + metadata: map[string]string{"sha256": digest}, + } + + pending := domain.Attachment{ + ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey, + StagingKey: stagingKey, ContentType: "image/png", + SizeBytes: int64(len(content)), SHA256: digest, + Status: domain.AttachmentPending, PipelineVersion: 1, + } + queued := domain.Attachment{ + ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: "attachments/" + chatID.String() + "/" + id.String(), + ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest, + Status: domain.AttachmentQueued, PipelineVersion: 1, + } + + repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once() + repo.EXPECT().CompletePresign( + mock.Anything, ownerID, id, mock.Anything, mock.Anything, + 1, 5, mock.Anything, + ).Return(domain.Attachment{}, domain.ErrConflict).Once() + repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(queued, nil).Once() + + got, err := service.Complete(t.Context(), ownerID, id, digest) + require.NoError(t, err) + require.Equal(t, domain.AttachmentQueued, got.Status) + + verified := 0 + for key := range store.objects { + if strings.HasPrefix(key, "staging/verified/") { + verified++ + } + } + require.Equal(t, 1, verified, "loser must not delete the winner's verified source") +} + +func TestPresignHonoursAdmissionLimit(t *testing.T) { + repo := mocks.NewMockRepository(t) + service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default()) + + repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once() + repo.EXPECT().CreatePresign(mock.Anything, mock.Anything, mock.Anything). + Return(domain.Attachment{}, domain.ErrRateLimited).Once() + + _, err := service.Presign(t.Context(), domain.PresignRequest{ + ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png", + ContentType: "image/png", SizeBytes: 10, SHA256: strings.Repeat("a", 64), + }) + require.ErrorIs(t, err, domain.ErrRateLimited) +} + +func TestUploadRateLimitedByAtomicAdmissionRemovesStaging(t *testing.T) { + repo := mocks.NewMockRepository(t) + store := newFakeStore() + service := attachment.NewService(repo, store, serviceConfig(), slog.Default()) + + repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once() + repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(0, 0, int64(0), nil).Once() + repo.EXPECT().CreateQueued(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(domain.Attachment{}, domain.ErrRateLimited).Once() + + _, err := service.Upload(t.Context(), attachment.UploadInput{ + ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png", + DeclaredContentType: "image/png", Body: bytes.NewReader(pngBytes(t, 8, 8)), + }) + require.ErrorIs(t, err, domain.ErrRateLimited) + require.Empty(t, store.objects, "a rejected atomic admission must not leave staging") +} + func TestCompleteRejectsChecksumMismatch(t *testing.T) { repo := mocks.NewMockRepository(t) store := newFakeStore() @@ -297,16 +451,13 @@ func TestCompleteRejectsChecksumMismatch(t *testing.T) { digest := hex.EncodeToString(sum[:]) stagingKey := "staging/presign/" + chatID.String() + "/" + id.String() - store.objects[stagingKey] = storedObject{ - data: content, - contentType: "image/png", - metadata: map[string]string{"sha256": digest}, - } + store.objects[stagingKey] = storedObject{data: content, contentType: "image/png", metadata: map[string]string{"sha256": digest}} pending := domain.Attachment{ ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey, - ContentType: "image/png", SizeBytes: int64(len(content)) + 1, SHA256: digest, - Status: domain.AttachmentPending, + StagingKey: stagingKey, ContentType: "image/png", + SizeBytes: int64(len(content)) + 1, SHA256: digest, + Status: domain.AttachmentPending, PipelineVersion: 1, } repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once() @@ -330,15 +481,12 @@ func TestCompleteKeepsObjectOnWrongRequestChecksum(t *testing.T) { digest := hex.EncodeToString(sum[:]) stagingKey := "staging/presign/" + chatID.String() + "/" + id.String() - store.objects[stagingKey] = storedObject{ - data: content, - contentType: "image/png", - metadata: map[string]string{"sha256": digest}, - } + store.objects[stagingKey] = storedObject{data: content, contentType: "image/png", metadata: map[string]string{"sha256": digest}} pending := domain.Attachment{ ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey, - ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest, - Status: domain.AttachmentPending, + StagingKey: stagingKey, ContentType: "image/png", + SizeBytes: int64(len(content)), SHA256: digest, + Status: domain.AttachmentPending, PipelineVersion: 1, } repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once() diff --git a/internal/service/attachment/sweeper.go b/internal/service/attachment/sweeper.go index 36eec1d6..5526df06 100644 --- a/internal/service/attachment/sweeper.go +++ b/internal/service/attachment/sweeper.go @@ -4,9 +4,10 @@ import ( "context" "errors" "log/slog" - "strings" "time" + "github.com/google/uuid" + "mtusi-messenger/internal/domain" "mtusi-messenger/internal/platform/objectstore" ) @@ -16,26 +17,38 @@ type LockStore interface { AcquireSweepLock(ctx context.Context) (release func(), acquired bool, err error) } +// SweepRepository is the storage port used by cleanup. It exposes only +// read/delete projections; object bytes still live in S3. +type SweepRepository interface { + ListPendingPresign(ctx context.Context, createdBefore time.Time, limit int) ([]domain.Attachment, error) + DeletePending(ctx context.Context, id uuid.UUID) error + ListStaleStaging(ctx context.Context, readyBefore, failedBefore time.Time, limit int) ([]domain.StaleStaging, error) + MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error + ListReferencedKeys(ctx context.Context, afterKey string, limit int) ([]string, error) +} + type SweepStats struct { - Recovered int RowsDeleted int ObjectsDeleted int + StagingCleared int } type Sweeper struct { - repo Repository - store objectstore.Store - locker LockStore - grace time.Duration - interval time.Duration - log *slog.Logger + repo SweepRepository + store objectstore.Store + locker LockStore + grace time.Duration + orphanGrace time.Duration + failedRetain time.Duration + interval time.Duration + log *slog.Logger } func NewSweeper( - repo Repository, + repo SweepRepository, store objectstore.Store, locker LockStore, - grace, interval time.Duration, + grace, orphanGrace, failedRetain, interval time.Duration, log *slog.Logger, ) *Sweeper { if log == nil { @@ -43,12 +56,14 @@ func NewSweeper( } return &Sweeper{ - repo: repo, - store: store, - locker: locker, - grace: grace, - interval: interval, - log: log, + repo: repo, + store: store, + locker: locker, + grace: grace, + orphanGrace: orphanGrace, + failedRetain: failedRetain, + interval: interval, + log: log, } } @@ -77,18 +92,18 @@ func (s *Sweeper) sweep(ctx context.Context) { return } - if stats.Recovered+stats.RowsDeleted+stats.ObjectsDeleted > 0 { + if stats.RowsDeleted+stats.ObjectsDeleted+stats.StagingCleared > 0 { s.log.InfoContext(ctx, "attachment sweep finished", - slog.Int("recovered", stats.Recovered), slog.Int("rows_deleted", stats.RowsDeleted), slog.Int("objects_deleted", stats.ObjectsDeleted), + slog.Int("staging_cleared", stats.StagingCleared), ) } } -// SweepOnce reconciles PostgreSQL metadata with S3 objects. It never deletes a -// ready row or an object referenced by any row. Only objects older than the -// grace period are candidates, so an in-flight upload cannot be collected. +// SweepOnce applies explicit, status-aware retention rules. It never deletes +// staging of a queued/processing attachment and aborts object collection if any +// database page or S3 listing fails. func (s *Sweeper) SweepOnce(ctx context.Context) (SweepStats, error) { release, acquired, err := s.locker.AcquireSweepLock(ctx) if err != nil { @@ -101,72 +116,20 @@ func (s *Sweeper) SweepOnce(ctx context.Context) (SweepStats, error) { defer release() - before := time.Now().UTC().Add(-s.grace) - + now := time.Now().UTC() var stats SweepStats - stale, err := s.repo.ListStalePending(ctx, before, 100) + // Abandoned presign rows: no completion ever arrived. + pending, err := s.repo.ListPendingPresign(ctx, now.Add(-s.grace), 100) if err != nil { return stats, err } - for _, row := range stale { - if row.Status != domain.AttachmentPending { - continue - } - - if strings.HasPrefix(row.ObjectKey, objectPrefixPresign) { - if s.deleteObject(ctx, row.ObjectKey) { + for _, row := range pending { + if row.StagingKey != "" { + if s.deleteObject(ctx, row.StagingKey) { stats.ObjectsDeleted++ } - - if err := s.repo.DeletePending(ctx, row.ID); err != nil && !errors.Is(err, domain.ErrNotFound) { - return stats, err - } - - stats.RowsDeleted++ - continue - } - - // A proxy upload writes directly to its final key. If the process died - // between PutObject and MarkReady, the object is complete and can be - // recovered instead of discarded. - head, err := s.store.Head(ctx, row.ObjectKey) - if err != nil { - if errors.Is(err, domain.ErrNotFound) { - if err := s.repo.DeletePending(ctx, row.ID); err != nil && !errors.Is(err, domain.ErrNotFound) { - return stats, err - } - - stats.RowsDeleted++ - continue - } - - s.log.WarnContext(ctx, "sweep head failed", - slog.String("attachment_id", row.ID.String()), - slog.Any("error", err), - ) - - continue - } - - if head.Size == row.SizeBytes && head.Metadata["sha256"] == row.SHA256 { - if _, err := s.repo.MarkReady(ctx, row.ID, row.ObjectKey); err != nil { - // A transient database failure must not destroy a valid object. - // Leave both the row and the object for the next sweep. - s.log.WarnContext(ctx, "sweep recovery failed", - slog.String("attachment_id", row.ID.String()), - slog.Any("error", err), - ) - continue - } - - stats.Recovered++ - continue - } - - if s.deleteObject(ctx, row.ObjectKey) { - stats.ObjectsDeleted++ } if err := s.repo.DeletePending(ctx, row.ID); err != nil && !errors.Is(err, domain.ErrNotFound) { @@ -176,12 +139,40 @@ func (s *Sweeper) SweepOnce(ctx context.Context) (SweepStats, error) { stats.RowsDeleted++ } + // Ready/failed rows keep their staging key until the object is safely + // removed. Active queued/processing staging is never in this list. + stale, err := s.repo.ListStaleStaging( + ctx, now.Add(-s.orphanGrace), now.Add(-s.failedRetain), 100, + ) + if err != nil { + return stats, err + } + + for _, row := range stale { + if s.deleteObject(ctx, row.StagingKey) { + stats.ObjectsDeleted++ + } + + if err := s.repo.MarkStagingDeleted(ctx, row.AttachmentID, row.StagingKey); err != nil { + return stats, err + } + + stats.StagingCleared++ + } + referenced, err := s.referencedKeys(ctx) if err != nil { return stats, err } - for _, prefix := range []string{objectPrefixAttachments, objectPrefixStaging} { + // Orphans: objects no row references. Final objects and variants use the + // normal grace; staging uses the longer orphan grace. + for _, prefix := range []string{domain.AttachmentsPrefix, "staging/"} { + before := now.Add(-s.grace) + if prefix == "staging/" { + before = now.Add(-s.orphanGrace) + } + objects, err := s.store.List(ctx, prefix) if err != nil { return stats, err @@ -209,20 +200,23 @@ func (s *Sweeper) referencedKeys(ctx context.Context) (map[string]struct{}, erro const pageSize = 1000 keys := make(map[string]struct{}) + after := "" - for offset := int32(0); ; offset += pageSize { - rows, err := s.repo.ListKeys(ctx, pageSize, offset) + for { + rows, err := s.repo.ListReferencedKeys(ctx, after, pageSize) if err != nil { return nil, err } - for _, row := range rows { - keys[row.ObjectKey] = struct{}{} + for _, key := range rows { + keys[key] = struct{}{} } if len(rows) < pageSize { return keys, nil } + + after = rows[len(rows)-1] } } diff --git a/internal/service/attachment/sweeper_test.go b/internal/service/attachment/sweeper_test.go index 8952b00e..51f8c89c 100644 --- a/internal/service/attachment/sweeper_test.go +++ b/internal/service/attachment/sweeper_test.go @@ -15,73 +15,71 @@ import ( "mtusi-messenger/internal/service/attachment/mocks" ) -func TestSweepReconcilesPendingAndOrphanObjects(t *testing.T) { - repo := mocks.NewMockRepository(t) +func TestSweepAppliesStatusAwareRetention(t *testing.T) { + repo := mocks.NewMockSweepRepository(t) locker := mocks.NewMockLockStore(t) store := newFakeStore() - stale := time.Now().UTC().Add(-time.Hour) + stale := time.Now().UTC().Add(-2 * time.Hour) presign := domain.Attachment{ - ID: uuid.New(), ObjectKey: "staging/presign/chat/" + uuid.NewString(), - Status: domain.AttachmentPending, CreatedAt: stale, + ID: uuid.New(), StagingKey: "staging/presign/chat/" + uuid.NewString(), + Status: domain.AttachmentPending, } - proxyRecoverable := domain.Attachment{ - ID: uuid.New(), ObjectKey: "attachments/chat/" + uuid.NewString(), - SizeBytes: 3, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - Status: domain.AttachmentPending, CreatedAt: stale, + staleStaging := domain.StaleStaging{ + AttachmentID: uuid.New(), StagingKey: "staging/uploads/chat/ready", Status: domain.AttachmentReady, } - proxyMissing := domain.Attachment{ - ID: uuid.New(), ObjectKey: "attachments/chat/" + uuid.NewString(), - Status: domain.AttachmentPending, CreatedAt: stale, - } - readyKey := "attachments/chat/ready" + activeStaging := "staging/uploads/chat/active" + readyKey := "attachments/chat/ready/original" + variantKey := "attachments/chat/ready/variants/v1/thumbnail.jpg" - store.objects[presign.ObjectKey] = storedObject{data: []byte("x"), lastModified: stale} - store.objects[proxyRecoverable.ObjectKey] = storedObject{ - data: []byte("abc"), - lastModified: stale, - metadata: map[string]string{"sha256": proxyRecoverable.SHA256}, + for _, key := range []string{ + presign.StagingKey, staleStaging.StagingKey, activeStaging, readyKey, variantKey, + "attachments/chat/orphan", "staging/orphan", + } { + store.objects[key] = storedObject{data: []byte("x"), lastModified: stale} } - store.objects[readyKey] = storedObject{data: []byte("ready"), lastModified: stale} - store.objects["attachments/chat/orphan"] = storedObject{data: []byte("orphan"), lastModified: stale} - store.objects["staging/abandoned"] = storedObject{data: []byte("stale"), lastModified: stale} - store.objects["staging/fresh"] = storedObject{data: []byte("fresh"), lastModified: time.Now().UTC()} + store.objects["staging/fresh"] = storedObject{data: []byte("x"), lastModified: time.Now().UTC()} - repo.EXPECT().ListStalePending(mock.Anything, mock.Anything, int32(100)). - Return([]domain.Attachment{presign, proxyRecoverable, proxyMissing}, nil).Once() + repo.EXPECT().ListPendingPresign(mock.Anything, mock.Anything, 100). + Return([]domain.Attachment{presign}, nil).Once() repo.EXPECT().DeletePending(mock.Anything, presign.ID).Return(nil).Once() - repo.EXPECT().MarkReady(mock.Anything, proxyRecoverable.ID, proxyRecoverable.ObjectKey). - Return(domain.Attachment{ID: proxyRecoverable.ID, Status: domain.AttachmentReady}, nil).Once() - repo.EXPECT().DeletePending(mock.Anything, proxyMissing.ID).Return(nil).Once() - repo.EXPECT().ListKeys(mock.Anything, int32(1000), int32(0)). - Return([]domain.AttachmentKey{ - {ID: proxyRecoverable.ID, ObjectKey: proxyRecoverable.ObjectKey, Status: domain.AttachmentReady}, - {ID: uuid.New(), ObjectKey: readyKey, Status: domain.AttachmentReady}, - }, nil).Once() - locker.EXPECT().AcquireSweepLock(mock.Anything). - Return(func() {}, true, nil).Once() + repo.EXPECT().ListStaleStaging(mock.Anything, mock.Anything, mock.Anything, 100). + Return([]domain.StaleStaging{staleStaging}, nil).Once() + repo.EXPECT().MarkStagingDeleted(mock.Anything, staleStaging.AttachmentID, staleStaging.StagingKey). + Return(nil).Once() + repo.EXPECT().ListReferencedKeys(mock.Anything, "", 1000). + Return([]string{activeStaging, readyKey, variantKey}, nil).Once() + locker.EXPECT().AcquireSweepLock(mock.Anything).Return(func() {}, true, nil).Once() - sweeper := attachment.NewSweeper(repo, store, locker, time.Minute, time.Hour, slog.Default()) + sweeper := attachment.NewSweeper( + repo, store, locker, + 10*time.Minute, time.Hour, 24*time.Hour, time.Minute, slog.Default(), + ) stats, err := sweeper.SweepOnce(t.Context()) require.NoError(t, err) - require.Equal(t, 1, stats.Recovered) - require.Equal(t, 2, stats.RowsDeleted) - require.Equal(t, 3, stats.ObjectsDeleted) + require.Equal(t, 1, stats.RowsDeleted) + require.Equal(t, 1, stats.StagingCleared) + require.Equal(t, 4, stats.ObjectsDeleted) - require.NotContains(t, store.objects, presign.ObjectKey) + require.NotContains(t, store.objects, presign.StagingKey) + require.NotContains(t, store.objects, staleStaging.StagingKey) require.NotContains(t, store.objects, "attachments/chat/orphan") - require.NotContains(t, store.objects, "staging/abandoned") - require.Contains(t, store.objects, proxyRecoverable.ObjectKey) + require.NotContains(t, store.objects, "staging/orphan") + require.Contains(t, store.objects, activeStaging) require.Contains(t, store.objects, readyKey) + require.Contains(t, store.objects, variantKey) require.Contains(t, store.objects, "staging/fresh") } func TestSweepSkipsWhenAnotherReplicaHoldsTheLock(t *testing.T) { - repo := mocks.NewMockRepository(t) + repo := mocks.NewMockSweepRepository(t) locker := mocks.NewMockLockStore(t) - sweeper := attachment.NewSweeper(repo, newFakeStore(), locker, time.Minute, time.Hour, slog.Default()) + sweeper := attachment.NewSweeper( + repo, newFakeStore(), locker, + time.Minute, time.Hour, 24*time.Hour, time.Minute, slog.Default(), + ) locker.EXPECT().AcquireSweepLock(mock.Anything).Return(nil, false, nil).Once() @@ -90,37 +88,26 @@ func TestSweepSkipsWhenAnotherReplicaHoldsTheLock(t *testing.T) { require.Zero(t, stats) } -func TestSweepKeepsObjectWhenMarkReadyFails(t *testing.T) { - repo := mocks.NewMockRepository(t) +func TestSweepAbortsObjectCollectionWhenKeysCannotBeRead(t *testing.T) { + repo := mocks.NewMockSweepRepository(t) locker := mocks.NewMockLockStore(t) store := newFakeStore() - stale := time.Now().UTC().Add(-time.Hour) - row := domain.Attachment{ - ID: uuid.New(), ObjectKey: "attachments/chat/" + uuid.NewString(), - SizeBytes: 3, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - Status: domain.AttachmentPending, CreatedAt: stale, - } + orphan := "attachments/chat/orphan" + store.objects[orphan] = storedObject{data: []byte("x"), lastModified: time.Now().UTC().Add(-2 * time.Hour)} - store.objects[row.ObjectKey] = storedObject{ - data: []byte("abc"), - lastModified: stale, - metadata: map[string]string{"sha256": row.SHA256}, - } - - repo.EXPECT().ListStalePending(mock.Anything, mock.Anything, int32(100)). - Return([]domain.Attachment{row}, nil).Once() - repo.EXPECT().MarkReady(mock.Anything, row.ID, row.ObjectKey). - Return(domain.Attachment{}, context.DeadlineExceeded).Once() - repo.EXPECT().ListKeys(mock.Anything, int32(1000), int32(0)). - Return([]domain.AttachmentKey{{ID: row.ID, ObjectKey: row.ObjectKey, Status: domain.AttachmentPending}}, nil).Once() + repo.EXPECT().ListPendingPresign(mock.Anything, mock.Anything, 100).Return(nil, nil).Once() + repo.EXPECT().ListStaleStaging(mock.Anything, mock.Anything, mock.Anything, 100).Return(nil, nil).Once() + repo.EXPECT().ListReferencedKeys(mock.Anything, "", 1000). + Return(nil, context.DeadlineExceeded).Once() locker.EXPECT().AcquireSweepLock(mock.Anything).Return(func() {}, true, nil).Once() - sweeper := attachment.NewSweeper(repo, store, locker, time.Minute, time.Hour, slog.Default()) + sweeper := attachment.NewSweeper( + repo, store, locker, + time.Minute, time.Hour, 24*time.Hour, time.Minute, slog.Default(), + ) - stats, err := sweeper.SweepOnce(t.Context()) - require.NoError(t, err) - require.Zero(t, stats.Recovered) - require.Zero(t, stats.RowsDeleted) - require.Contains(t, store.objects, row.ObjectKey) + _, err := sweeper.SweepOnce(t.Context()) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Contains(t, store.objects, orphan, "fail-closed: no deletion when the key set is unknown") } diff --git a/internal/service/message/mocks/Repository.go b/internal/service/message/mocks/Repository.go index f07fe488..369d2dc7 100644 --- a/internal/service/message/mocks/Repository.go +++ b/internal/service/message/mocks/Repository.go @@ -269,3 +269,75 @@ func (_c *MockRepository_ListVisible_Call) RunAndReturn(run func(ctx context.Con _c.Call.Return(run) return _c } + +// Replay provides a mock function for the type MockRepository +func (_mock *MockRepository) Replay(ctx context.Context, draft domain.MessageDraft) (domain.Message, bool, error) { + ret := _mock.Called(ctx, draft) + + if len(ret) == 0 { + panic("no return value specified for Replay") + } + + var r0 domain.Message + var r1 bool + var r2 error + if returnFunc, ok := ret.Get(0).(func(context.Context, domain.MessageDraft) (domain.Message, bool, error)); ok { + return returnFunc(ctx, draft) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, domain.MessageDraft) domain.Message); ok { + r0 = returnFunc(ctx, draft) + } else { + r0 = ret.Get(0).(domain.Message) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, domain.MessageDraft) bool); ok { + r1 = returnFunc(ctx, draft) + } else { + r1 = ret.Get(1).(bool) + } + if returnFunc, ok := ret.Get(2).(func(context.Context, domain.MessageDraft) error); ok { + r2 = returnFunc(ctx, draft) + } else { + r2 = ret.Error(2) + } + return r0, r1, r2 +} + +// MockRepository_Replay_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Replay' +type MockRepository_Replay_Call struct { + *mock.Call +} + +// Replay is a helper method to define mock.On call +// - ctx context.Context +// - draft domain.MessageDraft +func (_e *MockRepository_Expecter) Replay(ctx any, draft any) *MockRepository_Replay_Call { + return &MockRepository_Replay_Call{Call: _e.mock.On("Replay", ctx, draft)} +} + +func (_c *MockRepository_Replay_Call) Run(run func(ctx context.Context, draft domain.MessageDraft)) *MockRepository_Replay_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 domain.MessageDraft + if args[1] != nil { + arg1 = args[1].(domain.MessageDraft) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockRepository_Replay_Call) Return(message domain.Message, b bool, err error) *MockRepository_Replay_Call { + _c.Call.Return(message, b, err) + return _c +} + +func (_c *MockRepository_Replay_Call) RunAndReturn(run func(ctx context.Context, draft domain.MessageDraft) (domain.Message, bool, error)) *MockRepository_Replay_Call { + _c.Call.Return(run) + return _c +} diff --git a/internal/service/message/service.go b/internal/service/message/service.go index 7f5c12d0..45bc5895 100644 --- a/internal/service/message/service.go +++ b/internal/service/message/service.go @@ -11,6 +11,11 @@ import ( type Repository interface { IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) + // Replay returns the stored message for a live Idempotency-Key. It must be + // checked before attachment-state validation: a message that was accepted + // while the attachment was queued keeps replaying as 201 even after the + // attachment later fails. + Replay(ctx context.Context, draft domain.MessageDraft) (domain.Message, bool, error) Create(ctx context.Context, draft domain.MessageDraft) (domain.Message, error) ListVisible( ctx context.Context, @@ -50,6 +55,15 @@ func (s *Service) Send(ctx context.Context, draft domain.MessageDraft) (domain.M return domain.Message{}, fmt.Errorf("%w: chat is not visible", domain.ErrNotFound) } + replayed, found, err := s.messages.Replay(ctx, draft) + if err != nil { + return domain.Message{}, fmt.Errorf("replay message: %w", err) + } + + if found { + return replayed, nil + } + if len(draft.AttachmentIDs) > 0 { if s.attachments == nil { return domain.Message{}, fmt.Errorf("attachment checker is not configured: %w", domain.ErrUnavailable) @@ -65,7 +79,10 @@ func (s *Service) Send(ctx context.Context, draft domain.MessageDraft) (domain.M } for _, state := range states { - if state.Status != domain.AttachmentReady { + switch state.Status { + case domain.AttachmentQueued, domain.AttachmentProcessing, domain.AttachmentReady: + // Accepted work is linkable by contract. + default: return domain.Message{}, fmt.Errorf( "%w: attachment %s is %s", domain.ErrConflict, state.ID, state.Status, diff --git a/internal/service/message/service_test.go b/internal/service/message/service_test.go index 2f703d72..afa9f05c 100644 --- a/internal/service/message/service_test.go +++ b/internal/service/message/service_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/google/uuid" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "mtusi-messenger/internal/domain" @@ -38,6 +39,8 @@ func (s *ServiceSuite) SetupTest() { s.caller = uuid.New() s.chatID = uuid.New() s.now = time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + s.messages.EXPECT().Replay(mock.Anything, mock.Anything). + Return(domain.Message{}, false, nil).Maybe() } func (s *ServiceSuite) draft() domain.MessageDraft { @@ -167,6 +170,40 @@ func (s *ServiceSuite) TestSendAcceptsReadyAttachment() { s.Equal([]uuid.UUID{attachmentID}, got.AttachmentIDs) } +func (s *ServiceSuite) TestSendReplaysBeforeAttachmentValidation() { + draft := s.draft() + draft.AttachmentIDs = []uuid.UUID{uuid.New()} + stored := domain.Message{ + ID: uuid.New(), ChatID: draft.ChatID, SenderID: draft.SenderID, + Text: draft.Text, AttachmentIDs: draft.AttachmentIDs, CreatedAt: s.now, + } + + // The stored key is authoritative even if the attachment has since failed; + // ListStates must not be consulted on a replay. + s.messages.ExpectedCalls = nil + s.messages.EXPECT().IsChatMember(s.T().Context(), draft.ChatID, draft.SenderID).Return(true, nil).Once() + s.messages.EXPECT().Replay(s.T().Context(), draft).Return(stored, true, nil).Once() + + got, err := s.service.Send(s.T().Context(), draft) + s.Require().NoError(err) + s.Equal(stored, got) +} + +func (s *ServiceSuite) TestSendRejectsFailedAttachment() { + attachmentID := uuid.New() + draft := s.draft() + draft.AttachmentIDs = []uuid.UUID{attachmentID} + + s.messages.EXPECT().IsChatMember(s.T().Context(), draft.ChatID, draft.SenderID).Return(true, nil).Once() + s.attachments.EXPECT(). + ListStates(s.T().Context(), draft.ChatID, []uuid.UUID{attachmentID}). + Return([]domain.AttachmentState{{ID: attachmentID, Status: domain.AttachmentFailed}}, nil). + Once() + + _, err := s.service.Send(s.T().Context(), draft) + s.Require().ErrorIs(err, domain.ErrConflict) +} + func (s *ServiceSuite) TestSendAllowsAttachmentOnlyMessage() { attachmentID := uuid.New() draft := s.draft() diff --git a/internal/storage/postgres/attachment_repository.go b/internal/storage/postgres/attachment_repository.go deleted file mode 100644 index 3caa97e9..00000000 --- a/internal/storage/postgres/attachment_repository.go +++ /dev/null @@ -1,215 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - "time" - - "github.com/google/uuid" - - "mtusi-messenger/internal/domain" - "mtusi-messenger/internal/storage/postgres/internal/sqlcgen" -) - -type AttachmentRepository struct { - store *Store -} - -func (r *AttachmentRepository) IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - member, err := r.store.queries.IsChatMember(ctx, sqlcgen.IsChatMemberParams{ - ChatID: chatID, - UserID: userID, - }) - if err != nil { - return false, fmt.Errorf("check chat membership: %w", mapError(err)) - } - - return member, nil -} - -func (r *AttachmentRepository) Create(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - row, err := r.store.queries.CreateAttachment(ctx, sqlcgen.CreateAttachmentParams{ - ID: draft.ID, - ChatID: draft.ChatID, - OwnerID: draft.OwnerID, - ObjectKey: draft.ObjectKey, - Filename: draft.Filename, - ContentType: draft.ContentType, - SizeBytes: draft.SizeBytes, - Sha256: draft.SHA256, - }) - if err != nil { - return domain.Attachment{}, fmt.Errorf("create attachment: %w", mapError(err)) - } - - return mapAttachment(row), nil -} - -func (r *AttachmentRepository) MarkReady( - ctx context.Context, - id uuid.UUID, - objectKey string, -) (domain.Attachment, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - row, err := r.store.queries.MarkAttachmentReady(ctx, sqlcgen.MarkAttachmentReadyParams{ - ID: id, - ObjectKey: objectKey, - }) - if err != nil { - return domain.Attachment{}, fmt.Errorf("mark attachment ready: %w", mapError(err)) - } - - return mapAttachment(row), nil -} - -func (r *AttachmentRepository) DeletePending(ctx context.Context, id uuid.UUID) error { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - if err := r.store.queries.DeletePendingAttachment(ctx, id); err != nil { - return fmt.Errorf("delete pending attachment: %w", mapError(err)) - } - - return nil -} - -func (r *AttachmentRepository) GetForMember( - ctx context.Context, - userID, id uuid.UUID, -) (domain.Attachment, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - row, err := r.store.queries.GetAttachmentForMember(ctx, sqlcgen.GetAttachmentForMemberParams{ - ID: id, - UserID: userID, - }) - if err != nil { - return domain.Attachment{}, fmt.Errorf("get attachment for member: %w", mapError(err)) - } - - return mapAttachment(row), nil -} - -func (r *AttachmentRepository) GetForOwner( - ctx context.Context, - ownerID, id uuid.UUID, -) (domain.Attachment, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - row, err := r.store.queries.GetAttachmentForOwner(ctx, sqlcgen.GetAttachmentForOwnerParams{ - ID: id, - OwnerID: ownerID, - }) - if err != nil { - return domain.Attachment{}, fmt.Errorf("get attachment for owner: %w", mapError(err)) - } - - return mapAttachment(row), nil -} - -func (r *AttachmentRepository) ListStates( - ctx context.Context, - chatID uuid.UUID, - ids []uuid.UUID, -) ([]domain.AttachmentState, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - rows, err := r.store.queries.ListAttachmentStatesForChat(ctx, sqlcgen.ListAttachmentStatesForChatParams{ - ChatID: chatID, - Ids: ids, - }) - if err != nil { - return nil, fmt.Errorf("list attachment states: %w", mapError(err)) - } - - states := make([]domain.AttachmentState, 0, len(rows)) - for _, row := range rows { - states = append(states, domain.AttachmentState{ - ID: row.ID, - Status: domain.AttachmentStatus(row.Status), - }) - } - - return states, nil -} - -func (r *AttachmentRepository) ListStalePending( - ctx context.Context, - createdBefore time.Time, - limit int32, -) ([]domain.Attachment, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - rows, err := r.store.queries.ListStalePendingAttachments(ctx, sqlcgen.ListStalePendingAttachmentsParams{ - CreatedBefore: createdBefore, - RowLimit: limit, - }) - if err != nil { - return nil, fmt.Errorf("list stale attachments: %w", mapError(err)) - } - - return mapAll(rows, mapAttachment), nil -} - -func (r *AttachmentRepository) ListKeys( - ctx context.Context, - limit, offset int32, -) ([]domain.AttachmentKey, error) { - ctx, cancel := r.store.timeout(ctx) - defer cancel() - - rows, err := r.store.queries.ListAttachmentKeys(ctx, sqlcgen.ListAttachmentKeysParams{ - RowLimit: limit, - RowOffset: offset, - }) - if err != nil { - return nil, fmt.Errorf("list attachment keys: %w", mapError(err)) - } - - keys := make([]domain.AttachmentKey, 0, len(rows)) - for _, row := range rows { - keys = append(keys, domain.AttachmentKey{ - ID: row.ID, - ObjectKey: row.ObjectKey, - Status: domain.AttachmentStatus(row.Status), - CreatedAt: toUTCTime(row.CreatedAt), - }) - } - - return keys, nil -} - -func mapAttachment(row sqlcgen.Attachment) domain.Attachment { - attachment := domain.Attachment{ - ID: row.ID, - ChatID: row.ChatID, - OwnerID: row.OwnerID, - ObjectKey: row.ObjectKey, - Filename: row.Filename, - ContentType: row.ContentType, - SizeBytes: row.SizeBytes, - SHA256: row.Sha256, - Status: domain.AttachmentStatus(row.Status), - CreatedAt: toUTCTime(row.CreatedAt), - UpdatedAt: toUTCTime(row.UpdatedAt), - } - - if row.ErrorCode.Valid { - code := row.ErrorCode.String - attachment.ErrorCode = &code - } - - return attachment -} diff --git a/internal/storage/postgres/attachment_repository_test.go b/internal/storage/postgres/attachment_repository_test.go deleted file mode 100644 index 98bac411..00000000 --- a/internal/storage/postgres/attachment_repository_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package postgres_test - -import ( - "strings" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/suite" - - "mtusi-messenger/internal/domain" -) - -type AttachmentRepositorySuite struct { - BaseSuite -} - -func TestAttachmentRepositorySuite(t *testing.T) { - suite.Run(t, new(AttachmentRepositorySuite)) -} - -func (s *AttachmentRepositorySuite) TestLifecycleAndMessageLinks() { - alice := s.createUser("alice") - bob := s.createUser("bob") - eve := s.createUser("eve") - chat := s.createChat(alice, bob) - - id, err := uuid.NewV7() - s.Require().NoError(err) - - objectKey := "attachments/" + chat.ID.String() + "/" + id.String() - digest := strings.Repeat("a", 64) - - created, err := s.store.Attachments().Create(s.T().Context(), domain.AttachmentDraft{ - ID: id, - ChatID: chat.ID, - OwnerID: alice.ID, - ObjectKey: objectKey, - Filename: "photo.png", - ContentType: "image/png", - SizeBytes: 123, - SHA256: digest, - }) - s.Require().NoError(err) - s.Equal(domain.AttachmentPending, created.Status) - - visible, err := s.store.Attachments().GetForMember(s.T().Context(), bob.ID, id) - s.Require().NoError(err) - s.Equal(alice.ID, visible.OwnerID) - - _, err = s.store.Attachments().GetForMember(s.T().Context(), eve.ID, id) - s.Require().ErrorIs(err, domain.ErrNotFound) - - ready, err := s.store.Attachments().MarkReady(s.T().Context(), id, objectKey) - s.Require().NoError(err) - s.Equal(domain.AttachmentReady, ready.Status) - - states, err := s.store.Attachments().ListStates( - s.T().Context(), chat.ID, []uuid.UUID{id}, - ) - s.Require().NoError(err) - s.Require().Len(states, 1) - s.Equal(domain.AttachmentReady, states[0].Status) - - key := uuid.NewString() - draft := domain.MessageDraft{ - ChatID: chat.ID, - SenderID: alice.ID, - Text: "", - AttachmentIDs: []uuid.UUID{id}, - Idempotency: domain.MessageIdempotency{ - Key: key, - RequestHash: [32]byte{1}, - }, - } - - message, err := s.store.Messages().Create(s.T().Context(), draft) - s.Require().NoError(err) - s.Equal([]uuid.UUID{id}, message.AttachmentIDs) - - page, err := s.store.Messages().ListVisible(s.T().Context(), chat.ID, bob.ID, nil, 50) - s.Require().NoError(err) - s.Require().Len(page.Items, 1) - s.Equal([]uuid.UUID{id}, page.Items[0].AttachmentIDs) - s.Empty(page.Items[0].Text) - - replayed, err := s.store.Messages().Create(s.T().Context(), draft) - s.Require().NoError(err) - s.Equal(message.ID, replayed.ID) - s.Equal([]uuid.UUID{id}, replayed.AttachmentIDs) -} diff --git a/internal/storage/postgres/internal/sqlcgen/attachments.sql.go b/internal/storage/postgres/internal/sqlcgen/attachments.sql.go index e7dfbac2..4fa3568a 100644 --- a/internal/storage/postgres/internal/sqlcgen/attachments.sql.go +++ b/internal/storage/postgres/internal/sqlcgen/attachments.sql.go @@ -7,7 +7,6 @@ package sqlcgen import ( "context" - "time" "github.com/google/uuid" ) @@ -28,228 +27,6 @@ func (q *Queries) AttachToMessage(ctx context.Context, arg AttachToMessageParams return err } -const createAttachment = `-- name: CreateAttachment :one -INSERT INTO attachments ( - id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, status -) -VALUES ( - $1, - $2, - $3, - $4, - $5, - $6, - $7, - $8, - 'pending' -) -RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at -` - -type CreateAttachmentParams struct { - ID uuid.UUID - ChatID uuid.UUID - OwnerID uuid.UUID - ObjectKey string - Filename string - ContentType string - SizeBytes int64 - Sha256 string -} - -func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentParams) (Attachment, error) { - row := q.db.QueryRow(ctx, createAttachment, - arg.ID, - arg.ChatID, - arg.OwnerID, - arg.ObjectKey, - arg.Filename, - arg.ContentType, - arg.SizeBytes, - arg.Sha256, - ) - var i Attachment - err := row.Scan( - &i.ID, - &i.ChatID, - &i.OwnerID, - &i.ObjectKey, - &i.Filename, - &i.ContentType, - &i.SizeBytes, - &i.Sha256, - &i.Status, - &i.ErrorCode, - &i.CreatedAt, - &i.UpdatedAt, - ) - return i, err -} - -const deletePendingAttachment = `-- name: DeletePendingAttachment :exec -DELETE FROM attachments -WHERE id = $1 - AND status = 'pending' -` - -func (q *Queries) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error { - _, err := q.db.Exec(ctx, deletePendingAttachment, id) - return err -} - -const getAttachmentForMember = `-- name: GetAttachmentForMember :one -SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type, - a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at -FROM attachments a -JOIN chat_members cm - ON cm.chat_id = a.chat_id - AND cm.user_id = $1 -WHERE a.id = $2 -` - -type GetAttachmentForMemberParams struct { - UserID uuid.UUID - ID uuid.UUID -} - -func (q *Queries) GetAttachmentForMember(ctx context.Context, arg GetAttachmentForMemberParams) (Attachment, error) { - row := q.db.QueryRow(ctx, getAttachmentForMember, arg.UserID, arg.ID) - var i Attachment - err := row.Scan( - &i.ID, - &i.ChatID, - &i.OwnerID, - &i.ObjectKey, - &i.Filename, - &i.ContentType, - &i.SizeBytes, - &i.Sha256, - &i.Status, - &i.ErrorCode, - &i.CreatedAt, - &i.UpdatedAt, - ) - return i, err -} - -const getAttachmentForOwner = `-- name: GetAttachmentForOwner :one -SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type, - a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at -FROM attachments a -JOIN chat_members cm - ON cm.chat_id = a.chat_id - AND cm.user_id = a.owner_id -WHERE a.id = $1 - AND a.owner_id = $2 -` - -type GetAttachmentForOwnerParams struct { - ID uuid.UUID - OwnerID uuid.UUID -} - -func (q *Queries) GetAttachmentForOwner(ctx context.Context, arg GetAttachmentForOwnerParams) (Attachment, error) { - row := q.db.QueryRow(ctx, getAttachmentForOwner, arg.ID, arg.OwnerID) - var i Attachment - err := row.Scan( - &i.ID, - &i.ChatID, - &i.OwnerID, - &i.ObjectKey, - &i.Filename, - &i.ContentType, - &i.SizeBytes, - &i.Sha256, - &i.Status, - &i.ErrorCode, - &i.CreatedAt, - &i.UpdatedAt, - ) - return i, err -} - -const listAttachmentKeys = `-- name: ListAttachmentKeys :many -SELECT id, object_key, status, created_at -FROM attachments -ORDER BY id -LIMIT $2 OFFSET $1 -` - -type ListAttachmentKeysParams struct { - RowOffset int32 - RowLimit int32 -} - -type ListAttachmentKeysRow struct { - ID uuid.UUID - ObjectKey string - Status string - CreatedAt time.Time -} - -func (q *Queries) ListAttachmentKeys(ctx context.Context, arg ListAttachmentKeysParams) ([]ListAttachmentKeysRow, error) { - rows, err := q.db.Query(ctx, listAttachmentKeys, arg.RowOffset, arg.RowLimit) - if err != nil { - return nil, err - } - defer rows.Close() - items := []ListAttachmentKeysRow{} - for rows.Next() { - var i ListAttachmentKeysRow - if err := rows.Scan( - &i.ID, - &i.ObjectKey, - &i.Status, - &i.CreatedAt, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const listAttachmentStatesForChat = `-- name: ListAttachmentStatesForChat :many -SELECT id, status -FROM attachments -WHERE chat_id = $1 - AND id = ANY($2::uuid[]) -` - -type ListAttachmentStatesForChatParams struct { - ChatID uuid.UUID - Ids []uuid.UUID -} - -type ListAttachmentStatesForChatRow struct { - ID uuid.UUID - Status string -} - -func (q *Queries) ListAttachmentStatesForChat(ctx context.Context, arg ListAttachmentStatesForChatParams) ([]ListAttachmentStatesForChatRow, error) { - rows, err := q.db.Query(ctx, listAttachmentStatesForChat, arg.ChatID, arg.Ids) - if err != nil { - return nil, err - } - defer rows.Close() - items := []ListAttachmentStatesForChatRow{} - for rows.Next() { - var i ListAttachmentStatesForChatRow - if err := rows.Scan(&i.ID, &i.Status); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const listMessageAttachments = `-- name: ListMessageAttachments :many SELECT message_id, attachment_id FROM message_attachments @@ -281,87 +58,3 @@ func (q *Queries) ListMessageAttachments(ctx context.Context, messageIds []uuid. } return items, nil } - -const listStalePendingAttachments = `-- name: ListStalePendingAttachments :many -SELECT id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at -FROM attachments -WHERE status = 'pending' - AND created_at < $1 -ORDER BY created_at -LIMIT $2 -` - -type ListStalePendingAttachmentsParams struct { - CreatedBefore time.Time - RowLimit int32 -} - -func (q *Queries) ListStalePendingAttachments(ctx context.Context, arg ListStalePendingAttachmentsParams) ([]Attachment, error) { - rows, err := q.db.Query(ctx, listStalePendingAttachments, arg.CreatedBefore, arg.RowLimit) - if err != nil { - return nil, err - } - defer rows.Close() - items := []Attachment{} - for rows.Next() { - var i Attachment - if err := rows.Scan( - &i.ID, - &i.ChatID, - &i.OwnerID, - &i.ObjectKey, - &i.Filename, - &i.ContentType, - &i.SizeBytes, - &i.Sha256, - &i.Status, - &i.ErrorCode, - &i.CreatedAt, - &i.UpdatedAt, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const markAttachmentReady = `-- name: MarkAttachmentReady :one -UPDATE attachments -SET status = 'ready', - object_key = $1, - updated_at = now() -WHERE id = $2 - AND status = 'pending' -RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at -` - -type MarkAttachmentReadyParams struct { - ObjectKey string - ID uuid.UUID -} - -func (q *Queries) MarkAttachmentReady(ctx context.Context, arg MarkAttachmentReadyParams) (Attachment, error) { - row := q.db.QueryRow(ctx, markAttachmentReady, arg.ObjectKey, arg.ID) - var i Attachment - err := row.Scan( - &i.ID, - &i.ChatID, - &i.OwnerID, - &i.ObjectKey, - &i.Filename, - &i.ContentType, - &i.SizeBytes, - &i.Sha256, - &i.Status, - &i.ErrorCode, - &i.CreatedAt, - &i.UpdatedAt, - ) - return i, err -} diff --git a/internal/storage/postgres/internal/sqlcgen/idempotency.sql.go b/internal/storage/postgres/internal/sqlcgen/idempotency.sql.go index 3aecb901..4913a7bd 100644 --- a/internal/storage/postgres/internal/sqlcgen/idempotency.sql.go +++ b/internal/storage/postgres/internal/sqlcgen/idempotency.sql.go @@ -126,3 +126,32 @@ func (q *Queries) GetMessageIdempotency(ctx context.Context, arg GetMessageIdemp err := row.Scan(&i.MessageID, &i.RequestHash) return i, err } + +const lookupMessageIdempotency = `-- name: LookupMessageIdempotency :one +SELECT message_id, request_hash +FROM message_idempotency +WHERE actor_id = $1 + AND chat_id = $2 + AND key = $3 + AND expires_at > now() +` + +type LookupMessageIdempotencyParams struct { + ActorID uuid.UUID + ChatID uuid.UUID + Key string +} + +type LookupMessageIdempotencyRow struct { + MessageID pgtype.UUID + RequestHash []byte +} + +// Read-only replay lookup used before attachment-state validation. A live key +// returns the stored message; an expired key is ignored and may be claimed. +func (q *Queries) LookupMessageIdempotency(ctx context.Context, arg LookupMessageIdempotencyParams) (LookupMessageIdempotencyRow, error) { + row := q.db.QueryRow(ctx, lookupMessageIdempotency, arg.ActorID, arg.ChatID, arg.Key) + var i LookupMessageIdempotencyRow + err := row.Scan(&i.MessageID, &i.RequestHash) + return i, err +} diff --git a/internal/storage/postgres/internal/sqlcgen/models.go b/internal/storage/postgres/internal/sqlcgen/models.go index 9f5a2647..fc862a02 100644 --- a/internal/storage/postgres/internal/sqlcgen/models.go +++ b/internal/storage/postgres/internal/sqlcgen/models.go @@ -12,18 +12,62 @@ import ( ) type Attachment struct { - ID uuid.UUID - ChatID uuid.UUID - OwnerID uuid.UUID - ObjectKey string - Filename string - ContentType string - SizeBytes int64 - Sha256 string - Status string - ErrorCode pgtype.Text - CreatedAt time.Time - UpdatedAt time.Time + ID uuid.UUID + ChatID uuid.UUID + OwnerID uuid.UUID + ObjectKey string + Filename string + ContentType string + SizeBytes int64 + Sha256 string + Status string + ErrorCode pgtype.Text + CreatedAt time.Time + UpdatedAt time.Time + PipelineVersion int32 + StagingKey pgtype.Text + ReadyAt pgtype.Timestamptz + FailedAt pgtype.Timestamptz +} + +type AttachmentJob struct { + JobID uuid.UUID + AttachmentID uuid.UUID + PipelineVersion int32 + SourceKey string + Attempt int32 + MaxAttempts int32 + Status string + AvailableAt time.Time + LeaseOwner pgtype.Text + LeaseExpiresAt pgtype.Timestamptz + LastErrorCode pgtype.Text + ReplayCount int32 + CreatedAt time.Time + UpdatedAt time.Time + CompletedAt pgtype.Timestamptz +} + +type AttachmentJobEvent struct { + ID int64 + JobID uuid.UUID + Event string + Attempt pgtype.Int4 + Detail []byte + CreatedAt time.Time +} + +type AttachmentVariant struct { + AttachmentID uuid.UUID + PipelineVersion int32 + Name string + ContentType string + SizeBytes int64 + Width int32 + Height int32 + Sha256 string + ObjectKey string + CreatedAt time.Time } type Chat struct { @@ -62,6 +106,22 @@ type MessageIdempotency struct { ExpiresAt time.Time } +type OutboxMessage struct { + ID int64 + JobID uuid.UUID + Attempt int32 + Kind string + RoutingKey string + Payload []byte + AvailableAt time.Time + PublishedAt pgtype.Timestamptz + ClaimedBy pgtype.Text + ClaimedUntil pgtype.Timestamptz + PublishAttempts int32 + LastError pgtype.Text + CreatedAt time.Time +} + type Session struct { TokenHash []byte UserID uuid.UUID diff --git a/internal/storage/postgres/message_repository.go b/internal/storage/postgres/message_repository.go index 84ee5640..a80b1969 100644 --- a/internal/storage/postgres/message_repository.go +++ b/internal/storage/postgres/message_repository.go @@ -56,6 +56,61 @@ func (r *MessageRepository) Create(ctx context.Context, draft domain.MessageDraf return created, nil } +// Replay returns the stored message for a live Idempotency-Key without +// touching attachment state. The caller uses it before attachment validation so +// a replay stays 201 even if a linked attachment later failed. +func (r *MessageRepository) Replay( + ctx context.Context, + draft domain.MessageDraft, +) (domain.Message, bool, error) { + var ( + replayed domain.Message + found bool + ) + + err := r.store.RunInTx(ctx, func(queries *sqlcgen.Queries) error { + stored, err := queries.LookupMessageIdempotency(ctx, sqlcgen.LookupMessageIdempotencyParams{ + ActorID: draft.SenderID, + ChatID: draft.ChatID, + Key: draft.Idempotency.Key, + }) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return mapError(err) + } + + if !bytes.Equal(stored.RequestHash, draft.Idempotency.RequestHash[:]) { + return domain.ErrIdempotency + } + + if !stored.MessageID.Valid { + return errors.New("idempotency record without a message") + } + + row, err := queries.GetMessageByID(ctx, uuid.UUID(stored.MessageID.Bytes)) + if err != nil { + return mapError(err) + } + + replayed = mapMessage(row.ID, row.ChatID, row.SenderID, row.Text, row.CreatedAt) + replayed.AttachmentIDs, err = listAttachmentIDs(ctx, queries, row.ID) + if err != nil { + return err + } + + found = true + + return nil + }) + if err != nil { + return domain.Message{}, false, fmt.Errorf("replay idempotent message: %w", mapError(err)) + } + + return replayed, found, nil +} + // createIdempotent stores the message and its Idempotency-Key in one // transaction: either both are committed or neither, so a replay never observes // a claimed key whose message is missing. Claiming is a single statement, so diff --git a/internal/storage/postgres/migrations/00005_async_pipeline.sql b/internal/storage/postgres/migrations/00005_async_pipeline.sql new file mode 100644 index 00000000..0b764167 --- /dev/null +++ b/internal/storage/postgres/migrations/00005_async_pipeline.sql @@ -0,0 +1,150 @@ +-- +goose Up +-- Lab5 async attachment pipeline: durable intent (jobs + transactional outbox), +-- per-version results (variants), and operator audit (events). + +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK ( + status IN ('pending', 'queued', 'processing', 'ready', 'failed') +) NOT VALID; +ALTER TABLE attachments VALIDATE CONSTRAINT attachments_status_check; + +ALTER TABLE attachments DROP CONSTRAINT attachments_error_consistency; +ALTER TABLE attachments ADD CONSTRAINT attachments_error_consistency CHECK ( + (status IN ('ready', 'queued', 'processing') AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) OR + status = 'pending' +) NOT VALID; +ALTER TABLE attachments VALIDATE CONSTRAINT attachments_error_consistency; + +ALTER TABLE attachments + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 + CHECK (pipeline_version >= 1), + ADD COLUMN staging_key text, + ADD COLUMN ready_at timestamptz, + ADD COLUMN failed_at timestamptz; + +CREATE UNIQUE INDEX attachments_staging_key_key + ON attachments (staging_key) + WHERE staging_key IS NOT NULL; + +CREATE INDEX attachments_pipeline_status_idx + ON attachments (status, pipeline_version); + +-- One logical job per (attachment, pipeline version). job_id is stable across +-- redelivery; attempt is the managed attempt number (1-based). +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + source_key text NOT NULL, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 100), + status text NOT NULL CHECK ( + status IN ('queued', 'processing', 'ready', 'failed', 'superseded') + ), + available_at timestamptz NOT NULL DEFAULT now(), + lease_owner text, + lease_expires_at timestamptz, + last_error_code text, + replay_count integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + completed_at timestamptz, + + CONSTRAINT attachment_jobs_attachment_version_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_lease_consistency CHECK ( + (status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) OR + (status <> 'processing') + ) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (status, available_at); + +CREATE INDEX attachment_jobs_lease_idx + ON attachment_jobs (status, lease_expires_at); + +-- Delivery ledger. available_at is the only retry scheduler; RabbitMQ is the +-- durable transport. No UNIQUE(job_id, attempt): a graceful shutdown may +-- re-publish the same attempt, and duplicates are harmless (worker CAS). +CREATE TABLE outbox_messages ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + attempt integer NOT NULL CHECK (attempt >= 1), + kind text NOT NULL DEFAULT 'publish', + routing_key text NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + claimed_by text, + claimed_until timestamptz, + publish_attempts integer NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX outbox_due_idx + ON outbox_messages (available_at, id) + WHERE published_at IS NULL; + +CREATE INDEX outbox_claim_expiry_idx + ON outbox_messages (claimed_until) + WHERE published_at IS NULL AND claimed_by IS NOT NULL; + +-- Effect key (attachment_id, pipeline_version, name). The primary key is the +-- idempotency boundary for redelivery and versioned pipelines. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + content_type text NOT NULL CHECK ( + content_type IN ('image/jpeg', 'image/png', 'image/webp') + ), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE attachment_job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + event text NOT NULL, + attempt integer, + detail jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX attachment_job_events_job_idx + ON attachment_job_events (job_id, created_at); + +-- +goose Down +DROP TABLE attachment_job_events; +DROP TABLE attachment_variants; +DROP TABLE outbox_messages; +DROP TABLE attachment_jobs; + +DROP INDEX attachments_pipeline_status_idx; +DROP INDEX attachments_staging_key_key; + +ALTER TABLE attachments + DROP COLUMN failed_at, + DROP COLUMN ready_at, + DROP COLUMN staging_key, + DROP COLUMN pipeline_version; + +ALTER TABLE attachments DROP CONSTRAINT attachments_error_consistency; +ALTER TABLE attachments ADD CONSTRAINT attachments_error_consistency CHECK ( + (status = 'ready' AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) OR + status = 'pending' +) NOT VALID; + +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK ( + status IN ('pending', 'ready', 'failed') +) NOT VALID; diff --git a/internal/storage/postgres/pipeline_repository.go b/internal/storage/postgres/pipeline_repository.go new file mode 100644 index 00000000..98da8959 --- /dev/null +++ b/internal/storage/postgres/pipeline_repository.go @@ -0,0 +1,2061 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "mtusi-messenger/internal/domain" +) + +// PipelineRepository owns the durable job/outbox/variant state. Queries that +// need CAS semantics, FOR UPDATE SKIP LOCKED or multi-table transactions live +// here as explicit SQL instead of being spread across the service layer. +type PipelineRepository struct { + store *Store +} + +const attachmentReturning = `id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, status, error_code, object_key, COALESCE(staging_key, ''), pipeline_version, + created_at, updated_at, ready_at, failed_at` + +const jobReturning = `job_id, attachment_id, pipeline_version, source_key, attempt, + max_attempts, status, available_at, lease_owner, lease_expires_at, last_error_code, + replay_count, created_at, updated_at, completed_at` + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanAttachmentRow(row rowScanner) (domain.Attachment, error) { + var ( + attachment domain.Attachment + status string + errCode pgtype.Text + readyAt pgtype.Timestamptz + failedAt pgtype.Timestamptz + ) + + err := row.Scan( + &attachment.ID, + &attachment.ChatID, + &attachment.OwnerID, + &attachment.Filename, + &attachment.ContentType, + &attachment.SizeBytes, + &attachment.SHA256, + &status, + &errCode, + &attachment.ObjectKey, + &attachment.StagingKey, + &attachment.PipelineVersion, + &attachment.CreatedAt, + &attachment.UpdatedAt, + &readyAt, + &failedAt, + ) + if err != nil { + return domain.Attachment{}, err + } + + attachment.Status = domain.AttachmentStatus(status) + attachment.CreatedAt = toUTCTime(attachment.CreatedAt) + attachment.UpdatedAt = toUTCTime(attachment.UpdatedAt) + if errCode.Valid { + code := errCode.String + attachment.ErrorCode = &code + } + if readyAt.Valid { + value := readyAt.Time.UTC() + attachment.ReadyAt = &value + } + if failedAt.Valid { + value := failedAt.Time.UTC() + attachment.FailedAt = &value + } + + return attachment, nil +} + +func scanJobRow(row rowScanner) (domain.Job, error) { + var ( + job domain.Job + status string + owner pgtype.Text + leaseEnds pgtype.Timestamptz + errCode pgtype.Text + completed pgtype.Timestamptz + ) + + err := row.Scan( + &job.ID, + &job.AttachmentID, + &job.PipelineVersion, + &job.SourceKey, + &job.Attempt, + &job.MaxAttempts, + &status, + &job.AvailableAt, + &owner, + &leaseEnds, + &errCode, + &job.ReplayCount, + &job.CreatedAt, + &job.UpdatedAt, + &completed, + ) + if err != nil { + return domain.Job{}, err + } + + job.Status = domain.JobStatus(status) + job.AvailableAt = toUTCTime(job.AvailableAt) + job.CreatedAt = toUTCTime(job.CreatedAt) + job.UpdatedAt = toUTCTime(job.UpdatedAt) + if owner.Valid { + job.LeaseOwner = owner.String + } + if leaseEnds.Valid { + value := leaseEnds.Time.UTC() + job.LeaseExpiresAt = &value + } + if errCode.Valid { + code := errCode.String + job.LastErrorCode = &code + } + if completed.Valid { + value := completed.Time.UTC() + job.CompletedAt = &value + } + + return job, nil +} + +func (r *PipelineRepository) withTx(ctx context.Context, fn func(pgx.Tx) error) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + tx, err := r.store.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + + defer func() { _ = tx.Rollback(ctx) }() + + if err := fn(tx); err != nil { + return err + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit transaction: %w", err) + } + + return nil +} + +// IsChatMember mirrors the existing repository port used by the service. +func (r *PipelineRepository) IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + var member bool + + err := r.store.pool.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2 + )`, chatID, userID).Scan(&member) + if err != nil { + return false, fmt.Errorf("check chat membership: %w", mapError(err)) + } + + return member, nil +} + +func (r *PipelineRepository) GetForMember( + ctx context.Context, + userID, id uuid.UUID, +) (domain.Attachment, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + row := r.store.pool.QueryRow(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments a + WHERE a.id = $1 + AND EXISTS ( + SELECT 1 FROM chat_members cm + WHERE cm.chat_id = a.chat_id AND cm.user_id = $2 + )`, id, userID) + + attachment, err := scanAttachmentRow(row) + if err != nil { + return domain.Attachment{}, fmt.Errorf("get attachment for member: %w", mapError(err)) + } + + return attachment, nil +} + +func (r *PipelineRepository) GetForOwner( + ctx context.Context, + ownerID, id uuid.UUID, +) (domain.Attachment, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + row := r.store.pool.QueryRow(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments a + WHERE a.id = $1 AND a.owner_id = $2`, id, ownerID) + + attachment, err := scanAttachmentRow(row) + if err != nil { + return domain.Attachment{}, fmt.Errorf("get attachment for owner: %w", mapError(err)) + } + + return attachment, nil +} + +func (r *PipelineRepository) ListStates( + ctx context.Context, + chatID uuid.UUID, + ids []uuid.UUID, +) ([]domain.AttachmentState, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + SELECT id, status + FROM attachments + WHERE chat_id = $1 AND id = ANY($2::uuid[])`, chatID, ids) + if err != nil { + return nil, fmt.Errorf("list attachment states: %w", mapError(err)) + } + defer rows.Close() + + states := make([]domain.AttachmentState, 0, len(ids)) + + for rows.Next() { + var ( + state domain.AttachmentState + status string + ) + if err := rows.Scan(&state.ID, &status); err != nil { + return nil, fmt.Errorf("scan attachment state: %w", mapError(err)) + } + state.Status = domain.AttachmentStatus(status) + states = append(states, state) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list attachment states: %w", mapError(err)) + } + + return states, nil +} + +func (r *PipelineRepository) ListVariants( + ctx context.Context, + attachmentID uuid.UUID, + pipelineVersion int, +) ([]domain.Variant, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY name`, attachmentID, pipelineVersion) + if err != nil { + return nil, fmt.Errorf("list variants: %w", mapError(err)) + } + defer rows.Close() + + variants := make([]domain.Variant, 0, 3) + + for rows.Next() { + var ( + variant domain.Variant + name string + ) + if err := rows.Scan( + &name, &variant.ContentType, &variant.SizeBytes, + &variant.Width, &variant.Height, &variant.SHA256, &variant.ObjectKey, + ); err != nil { + return nil, fmt.Errorf("scan variant: %w", mapError(err)) + } + variant.Name = domain.VariantName(name) + variants = append(variants, variant) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list variants: %w", mapError(err)) + } + + return variants, nil +} + +func (r *PipelineRepository) GetVariant( + ctx context.Context, + attachmentID uuid.UUID, + pipelineVersion int, + name domain.VariantName, +) (domain.Variant, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + var variant domain.Variant + + err := r.store.pool.QueryRow(ctx, ` + SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 AND name = $3`, + attachmentID, pipelineVersion, string(name)).Scan( + &variant.Name, &variant.ContentType, &variant.SizeBytes, + &variant.Width, &variant.Height, &variant.SHA256, &variant.ObjectKey, + ) + if err != nil { + return domain.Variant{}, fmt.Errorf("get variant: %w", mapError(err)) + } + + return variant, nil +} + +func (r *PipelineRepository) Admission( + ctx context.Context, + ownerID uuid.UUID, +) (userPending, totalPending int, stagedBytes int64, err error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + err = r.store.pool.QueryRow(ctx, ` + SELECT + count(*) FILTER (WHERE owner_id = $1)::int, + count(*)::int, + COALESCE(sum(size_bytes), 0)::bigint + FROM attachments + WHERE status IN ('pending', 'queued', 'processing')`, ownerID).Scan( + &userPending, &totalPending, &stagedBytes, + ) + if err != nil { + return 0, 0, 0, fmt.Errorf("admission counts: %w", mapError(err)) + } + + return userPending, totalPending, stagedBytes, nil +} + +func insertOutboxTx( + ctx context.Context, + tx pgx.Tx, + job domain.Job, + attempt int, + kind string, + availableAt time.Time, +) error { + payload, err := domain.NewJobMessage(domain.Job{ + ID: job.ID, + AttachmentID: job.AttachmentID, + PipelineVersion: job.PipelineVersion, + Attempt: attempt, + CreatedAt: job.CreatedAt, + }).MarshalPayload() + if err != nil { + return fmt.Errorf("marshal job message: %w", err) + } + + _, err = tx.Exec(ctx, ` + INSERT INTO outbox_messages (job_id, attempt, kind, routing_key, payload, available_at) + VALUES ($1, $2, $3, $4, $5::jsonb, $6)`, + job.ID, attempt, kind, defaultRoutingKey, string(payload), availableAt) + if err != nil { + return fmt.Errorf("insert outbox: %w", err) + } + + return nil +} + +func insertEventTx(ctx context.Context, tx pgx.Tx, jobID uuid.UUID, event string, attempt int, detail string) error { + _, err := tx.Exec(ctx, ` + INSERT INTO attachment_job_events (job_id, event, attempt, detail) + VALUES ($1, $2, $3, NULLIF($4, '')::jsonb)`, jobID, event, attempt, detail) + if err != nil { + return fmt.Errorf("insert job event: %w", err) + } + + return nil +} + +const defaultRoutingKey = "attachment.process.v1" + +// CreateQueued persists the attachment, its job and the outbox intent in one +// transaction. The caller has already written the durable staging object. +func (r *PipelineRepository) CreateQueued( + ctx context.Context, + draft domain.AttachmentDraft, + job domain.Job, + limits domain.AdmissionLimits, +) (domain.Attachment, error) { + var created domain.Attachment + + err := r.withTx(ctx, func(tx pgx.Tx) error { + if err := checkAdmissionTx(ctx, tx, draft.OwnerID, limits); err != nil { + return err + } + + row := tx.QueryRow(ctx, ` + INSERT INTO attachments ( + id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, status, pipeline_version, staging_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', $9, $10) + RETURNING `+attachmentReturning, + draft.ID, draft.ChatID, draft.OwnerID, draft.ObjectKey, draft.Filename, + draft.ContentType, draft.SizeBytes, draft.SHA256, draft.PipelineVersion, + draft.StagingKey, + ) + + attachment, err := scanAttachmentRow(row) + if err != nil { + return fmt.Errorf("create attachment: %w", mapError(err)) + } + + _, err = tx.Exec(ctx, ` + INSERT INTO attachment_jobs ( + job_id, attachment_id, pipeline_version, source_key, + attempt, max_attempts, status, available_at + ) + VALUES ($1, $2, $3, $4, 1, $5, 'queued', now())`, + job.ID, draft.ID, job.PipelineVersion, job.SourceKey, job.MaxAttempts) + if err != nil { + return fmt.Errorf("create job: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventEnqueued, 1, ""); err != nil { + return err + } + + if err := insertOutboxTx( + ctx, tx, job, 1, "initial", time.Now().UTC(), + ); err != nil { + return err + } + + created = attachment + + return nil + }) + if err != nil { + return domain.Attachment{}, err + } + + return created, nil +} + +// CompletePresign moves a verified presign object into the queue. +func (r *PipelineRepository) CompletePresign( + ctx context.Context, + ownerID, attachmentID uuid.UUID, + verifiedKey, finalKey string, + pipelineVersion, maxAttempts int, + jobID uuid.UUID, +) (domain.Attachment, error) { + var completed domain.Attachment + + err := r.withTx(ctx, func(tx pgx.Tx) error { + row := tx.QueryRow(ctx, ` + UPDATE attachments + SET status = 'queued', + staging_key = $3, + object_key = $4, + pipeline_version = $5, + error_code = NULL, + failed_at = NULL, + updated_at = now() + WHERE id = $1 AND owner_id = $2 AND status = 'pending' + RETURNING `+attachmentReturning, + attachmentID, ownerID, verifiedKey, finalKey, pipelineVersion, + ) + + attachment, err := scanAttachmentRow(row) + if err != nil { + if errors.Is(mapError(err), domain.ErrNotFound) { + return fmt.Errorf("%w: attachment is not pending", domain.ErrConflict) + } + + return fmt.Errorf("queue presigned attachment: %w", mapError(err)) + } + + job := domain.Job{ + ID: jobID, + AttachmentID: attachmentID, + PipelineVersion: pipelineVersion, + SourceKey: verifiedKey, + Attempt: 1, + MaxAttempts: maxAttempts, + CreatedAt: time.Now().UTC(), + } + + _, err = tx.Exec(ctx, ` + INSERT INTO attachment_jobs ( + job_id, attachment_id, pipeline_version, source_key, + attempt, max_attempts, status, available_at + ) + VALUES ($1, $2, $3, $4, 1, $5, 'queued', now())`, + job.ID, job.AttachmentID, job.PipelineVersion, job.SourceKey, job.MaxAttempts) + if err != nil { + return fmt.Errorf("create presign job: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventEnqueued, 1, ""); err != nil { + return err + } + + if err := insertOutboxTx( + ctx, tx, job, 1, "initial", time.Now().UTC(), + ); err != nil { + return err + } + + completed = attachment + + return nil + }) + if err != nil { + return domain.Attachment{}, err + } + + return completed, nil +} + +// ClaimJob atomically turns one delivery into the active processing attempt. +// The returned outcome is authoritative; the worker never guesses from message +// fields alone. +func (r *PipelineRepository) ClaimJob( + ctx context.Context, + message domain.JobMessage, + leaseOwner string, + leaseTTL time.Duration, +) (domain.ClaimResult, error) { + var result domain.ClaimResult + + err := r.withTx(ctx, func(tx pgx.Tx) error { + jobRow := tx.QueryRow(ctx, ` + SELECT `+jobReturning+` + FROM attachment_jobs + WHERE job_id = $1 + FOR UPDATE`, message.JobID) + + job, err := scanJobRow(jobRow) + if err != nil { + if errors.Is(mapError(err), domain.ErrNotFound) { + result = domain.ClaimResult{Outcome: domain.ClaimJobNotFound} + + return nil + } + + return fmt.Errorf("load job: %w", mapError(err)) + } + + result.Job = job + + attachmentRow := tx.QueryRow(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments + WHERE id = $1 + FOR UPDATE`, job.AttachmentID) + + attachment, err := scanAttachmentRow(attachmentRow) + if err != nil { + return fmt.Errorf("load job attachment: %w", mapError(err)) + } + + result.Attachment = attachment + + switch { + case job.PipelineVersion < attachment.PipelineVersion: + // A queued job left behind by an older version (for example a + // replay accepted before the newer version committed) can never + // make progress. Retire it instead of acknowledging and + // republishing it through reconciliation forever. + if job.Status == domain.JobQueued { + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'superseded', updated_at = now(), completed_at = now() + WHERE job_id = $1 AND status = 'queued'`, job.ID); err != nil { + return fmt.Errorf("supersede version-stale job: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventSuperseded, job.Attempt, ""); err != nil { + return err + } + } + + result.Outcome = domain.ClaimVersionStale + + return nil + case job.Status == domain.JobReady: + result.Outcome = domain.ClaimAlreadyReady + + return nil + case job.Status == domain.JobFailed: + result.Outcome = domain.ClaimAlreadyFailed + + return nil + case job.Status == domain.JobSuperseded: + result.Outcome = domain.ClaimSuperseded + + return nil + case message.Attempt < job.Attempt: + result.Outcome = domain.ClaimStaleAttempt + + return nil + case message.Attempt > job.Attempt: + result.Outcome = domain.ClaimFutureAttempt + + return nil + case job.Status == domain.JobProcessing: + // A live lease owns the effect; an expired one is requeued by the + // reaper, not by a duplicate delivery. + result.Outcome = domain.ClaimBusyLease + + return nil + case job.Status != domain.JobQueued: + result.Outcome = domain.ClaimJobNotFound + + return nil + } + + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'processing', + lease_owner = $2, + lease_expires_at = now() + $3::interval, + updated_at = now() + WHERE job_id = $1 AND status = 'queued' AND attempt = $4 + AND available_at <= now()`, + job.ID, leaseOwner, intervalString(leaseTTL), message.Attempt) + if err != nil { + return fmt.Errorf("claim job: %w", mapError(err)) + } + + if tag.RowsAffected() == 0 { + result.Outcome = domain.ClaimNotDue + + return nil + } + + if job.PipelineVersion == attachment.PipelineVersion && attachment.Status == domain.AttachmentQueued { + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'processing', updated_at = now() + WHERE id = $1 AND status = 'queued' AND pipeline_version = $2`, + attachment.ID, job.PipelineVersion) + if err != nil { + return fmt.Errorf("mark attachment processing: %w", mapError(err)) + } + + attachment.Status = domain.AttachmentProcessing + result.Attachment = attachment + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventClaimed, message.Attempt, ""); err != nil { + return err + } + + job.Status = domain.JobProcessing + job.LeaseOwner = leaseOwner + expires := time.Now().Add(leaseTTL).UTC() + job.LeaseExpiresAt = &expires + result.Job = job + result.Outcome = domain.ClaimAcquired + + return nil + }) + if err != nil { + return domain.ClaimResult{}, err + } + + return result, nil +} + +func intervalString(d time.Duration) string { + return fmt.Sprintf("%d milliseconds", d.Milliseconds()) +} + +func (r *PipelineRepository) Heartbeat( + ctx context.Context, + jobID uuid.UUID, + attempt int, + leaseOwner string, + leaseTTL time.Duration, +) (bool, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + tag, err := r.store.pool.Exec(ctx, ` + UPDATE attachment_jobs + SET lease_expires_at = now() + $4::interval, updated_at = now() + WHERE job_id = $1 AND attempt = $2 AND lease_owner = $3 + AND status = 'processing' AND lease_expires_at > now()`, + jobID, attempt, leaseOwner, intervalString(leaseTTL)) + if err != nil { + return false, fmt.Errorf("heartbeat: %w", mapError(err)) + } + + return tag.RowsAffected() == 1, nil +} + +// CompleteJob inserts the effect rows and publishes the result atomically. +// A newer serving pipeline version is never downgraded. +func (r *PipelineRepository) CompleteJob( + ctx context.Context, + jobID uuid.UUID, + attempt int, + leaseOwner string, + variants []domain.Variant, +) (bool, error) { + completed := false + + err := r.withTx(ctx, func(tx pgx.Tx) error { + job, attachment, err := lockJobAndAttachment(ctx, tx, jobID) + if err != nil { + return err + } + + if job.Status != domain.JobProcessing || job.Attempt != attempt || job.LeaseOwner != leaseOwner { + return nil + } + + if job.PipelineVersion < attachment.PipelineVersion { + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'superseded', updated_at = now(), completed_at = now() + WHERE job_id = $1`, job.ID); err != nil { + return fmt.Errorf("supersede stale job: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventSuperseded, attempt, ""); err != nil { + return err + } + + return nil + } + + for _, variant := range variants { + tag, err := tx.Exec(ctx, ` + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING`, + job.AttachmentID, job.PipelineVersion, string(variant.Name), + variant.ContentType, variant.SizeBytes, variant.Width, variant.Height, + variant.SHA256, variant.ObjectKey) + if err != nil { + return fmt.Errorf("insert variant: %w", mapError(err)) + } + + if tag.RowsAffected() == 0 { + existing, err := getVariantTx(ctx, tx, job.AttachmentID, job.PipelineVersion, variant.Name) + if err != nil { + return err + } + + if existing.ContentType != variant.ContentType || + existing.SizeBytes != variant.SizeBytes || + existing.Width != variant.Width || + existing.Height != variant.Height || + existing.SHA256 != variant.SHA256 || + existing.ObjectKey != variant.ObjectKey { + return fmt.Errorf("%w: variant %s already has different bytes", domain.ErrConflict, variant.Name) + } + } + } + + if job.PipelineVersion > attachment.PipelineVersion { + // Reprocess completed: atomically switch the published version. + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET pipeline_version = $2, + status = 'ready', + error_code = NULL, + ready_at = now(), + failed_at = NULL, + updated_at = now() + WHERE id = $1`, attachment.ID, job.PipelineVersion) + } else { + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'ready', + error_code = NULL, + ready_at = now(), + failed_at = NULL, + updated_at = now() + WHERE id = $1`, attachment.ID) + } + if err != nil { + return fmt.Errorf("mark attachment ready: %w", mapError(err)) + } + + _, err = tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'ready', + lease_owner = NULL, + lease_expires_at = NULL, + last_error_code = NULL, + completed_at = now(), + updated_at = now() + WHERE job_id = $1`, job.ID) + if err != nil { + return fmt.Errorf("mark job ready: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventReady, attempt, ""); err != nil { + return err + } + + completed = true + + return nil + }) + if err != nil { + return false, err + } + + return completed, nil +} + +func lockJobAndAttachment( + ctx context.Context, + tx pgx.Tx, + jobID uuid.UUID, +) (domain.Job, domain.Attachment, error) { + jobRow := tx.QueryRow(ctx, ` + SELECT `+jobReturning+` + FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID) + + job, err := scanJobRow(jobRow) + if err != nil { + return domain.Job{}, domain.Attachment{}, fmt.Errorf("load job: %w", mapError(err)) + } + + attachmentRow := tx.QueryRow(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments WHERE id = $1 FOR UPDATE`, job.AttachmentID) + + attachment, err := scanAttachmentRow(attachmentRow) + if err != nil { + return domain.Job{}, domain.Attachment{}, fmt.Errorf("load attachment: %w", mapError(err)) + } + + return job, attachment, nil +} + +func getVariantTx( + ctx context.Context, + tx pgx.Tx, + attachmentID uuid.UUID, + pipelineVersion int, + name domain.VariantName, +) (domain.Variant, error) { + var variant domain.Variant + + err := tx.QueryRow(ctx, ` + SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 AND name = $3`, + attachmentID, pipelineVersion, string(name)).Scan( + &variant.Name, &variant.ContentType, &variant.SizeBytes, + &variant.Width, &variant.Height, &variant.SHA256, &variant.ObjectKey, + ) + if err != nil { + return domain.Variant{}, fmt.Errorf("load variant: %w", mapError(err)) + } + + return variant, nil +} + +// RetryJob schedules the next managed attempt. sameAttempt is used by graceful +// shutdown: infrastructure restarts must not consume the processing budget. +func (r *PipelineRepository) RetryJob( + ctx context.Context, + jobID uuid.UUID, + attempt int, + leaseOwner string, + errorCode string, + nextAvailable time.Time, + sameAttempt bool, + kind string, +) (failed bool, err error) { + err = r.withTx(ctx, func(tx pgx.Tx) error { + job, attachment, err := lockJobAndAttachment(ctx, tx, jobID) + if err != nil { + return err + } + + if job.Status != domain.JobProcessing || job.Attempt != attempt || job.LeaseOwner != leaseOwner { + return nil + } + + if !sameAttempt && job.Attempt >= job.MaxAttempts { + failed = true + + return failJobTx(ctx, tx, job, attachment, "retry_exhausted", false) + } + + nextAttempt := job.Attempt + if !sameAttempt { + nextAttempt++ + } + + _, err = tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'queued', + attempt = $2, + available_at = $3, + lease_owner = NULL, + lease_expires_at = NULL, + last_error_code = $4, + updated_at = now() + WHERE job_id = $1`, + job.ID, nextAttempt, nextAvailable, errorCode) + if err != nil { + return fmt.Errorf("schedule retry: %w", mapError(err)) + } + + if job.PipelineVersion == attachment.PipelineVersion && + (attachment.Status == domain.AttachmentProcessing || attachment.Status == domain.AttachmentQueued) { + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', updated_at = now() + WHERE id = $1 AND pipeline_version = $2`, + attachment.ID, job.PipelineVersion) + if err != nil { + return fmt.Errorf("requeue attachment: %w", mapError(err)) + } + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventRetried, nextAttempt, `{"error_code":"`+errorCode+`"}`); err != nil { + return err + } + + job.Attempt = nextAttempt + + if err := insertOutboxTx(ctx, tx, job, nextAttempt, kind, nextAvailable); err != nil { + return err + } + + return nil + }) + if err != nil { + return false, err + } + + return failed, nil +} + +func (r *PipelineRepository) FailJob( + ctx context.Context, + jobID uuid.UUID, + attempt int, + leaseOwner string, + errorCode string, +) (bool, error) { + failed := false + + err := r.withTx(ctx, func(tx pgx.Tx) error { + job, attachment, err := lockJobAndAttachment(ctx, tx, jobID) + if err != nil { + return err + } + + if job.Status != domain.JobProcessing || job.Attempt != attempt || job.LeaseOwner != leaseOwner { + return nil + } + + if err := failJobTx(ctx, tx, job, attachment, errorCode, false); err != nil { + return err + } + + failed = true + + return nil + }) + if err != nil { + return false, err + } + + return failed, nil +} + +func failJobTx( + ctx context.Context, + tx pgx.Tx, + job domain.Job, + attachment domain.Attachment, + errorCode string, + leaseExpired bool, +) error { + _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'failed', + lease_owner = NULL, + lease_expires_at = NULL, + last_error_code = $2, + completed_at = now(), + updated_at = now() + WHERE job_id = $1`, job.ID, errorCode) + if err != nil { + return fmt.Errorf("mark job failed: %w", mapError(err)) + } + + // A failed job settles the attachment when its version is the serving one, + // or when it is a newer version attempted before any version was + // published (reprocess from queued/processing/failed). A newer version + // failing while an older ready version is serving must not disturb the + // published version: it stays ready and downloadable. + settlesAttachment := job.PipelineVersion == attachment.PipelineVersion || + (job.PipelineVersion > attachment.PipelineVersion && attachment.Status != domain.AttachmentReady) + + if settlesAttachment { + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'failed', + error_code = $2, + failed_at = now(), + updated_at = now() + WHERE id = $1`, attachment.ID, errorCode) + if err != nil { + return fmt.Errorf("mark attachment failed: %w", mapError(err)) + } + } + + detail := `{"error_code":"` + errorCode + `"}` + + if leaseExpired { + if err := insertEventTx(ctx, tx, job.ID, domain.EventLeaseExpired, job.Attempt, detail); err != nil { + return err + } + } + + return insertEventTx(ctx, tx, job.ID, domain.EventFailed, job.Attempt, detail) +} + +// ReapExpiredLeases requeues dead workers and terminates attempts that already +// exhausted their budget. The second branch is what makes retry bounded. +func (r *PipelineRepository) ReapExpiredLeases( + ctx context.Context, + limit int, +) (int, error) { + reaped := 0 + + err := r.withTx(ctx, func(tx pgx.Tx) error { + rows, err := tx.Query(ctx, ` + SELECT `+jobReturning+` + FROM attachment_jobs + WHERE status = 'processing' AND lease_expires_at < now() + ORDER BY lease_expires_at + LIMIT $1 + FOR UPDATE SKIP LOCKED`, limit) + if err != nil { + return fmt.Errorf("select expired leases: %w", mapError(err)) + } + + jobs := make([]domain.Job, 0, limit) + + for rows.Next() { + job, err := scanJobRow(rows) + if err != nil { + rows.Close() + + return fmt.Errorf("scan expired lease: %w", mapError(err)) + } + jobs = append(jobs, job) + } + + if err := rows.Err(); err != nil { + rows.Close() + + return fmt.Errorf("list expired leases: %w", mapError(err)) + } + rows.Close() + + for _, job := range jobs { + attachmentRow := tx.QueryRow(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments WHERE id = $1 FOR UPDATE`, job.AttachmentID) + + attachment, err := scanAttachmentRow(attachmentRow) + if err != nil { + return fmt.Errorf("load lease attachment: %w", mapError(err)) + } + + if job.Attempt >= job.MaxAttempts { + if err := failJobTx(ctx, tx, job, attachment, "retry_exhausted", true); err != nil { + return err + } + reaped++ + + continue + } + + nextAttempt := job.Attempt + 1 + + _, err = tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'queued', + attempt = $2, + available_at = now(), + lease_owner = NULL, + lease_expires_at = NULL, + last_error_code = 'lease_expired', + updated_at = now() + WHERE job_id = $1`, job.ID, nextAttempt) + if err != nil { + return fmt.Errorf("reap lease: %w", mapError(err)) + } + + if job.PipelineVersion == attachment.PipelineVersion && + (attachment.Status == domain.AttachmentProcessing || attachment.Status == domain.AttachmentQueued) { + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', updated_at = now() + WHERE id = $1 AND pipeline_version = $2`, + attachment.ID, job.PipelineVersion) + if err != nil { + return fmt.Errorf("requeue reaped attachment: %w", mapError(err)) + } + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventLeaseExpired, job.Attempt, ""); err != nil { + return err + } + + job.Attempt = nextAttempt + job.AvailableAt = time.Now().UTC() + + if err := insertOutboxTx( + ctx, tx, job, nextAttempt, "lease_expired", time.Now().UTC(), + ); err != nil { + return err + } + + reaped++ + } + + return nil + }) + if err != nil { + return 0, err + } + + return reaped, nil +} + +func (r *PipelineRepository) ClaimOutbox( + ctx context.Context, + owner string, + claimTTL time.Duration, + limit int, +) ([]domain.OutboxMessage, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + WITH due AS ( + SELECT o.id + FROM outbox_messages o + WHERE o.published_at IS NULL + AND o.available_at <= now() + AND (o.claimed_until IS NULL OR o.claimed_until < now()) + AND NOT EXISTS ( + SELECT 1 FROM outbox_messages p + WHERE p.job_id = o.job_id + AND p.published_at IS NULL + AND p.id < o.id + ) + ORDER BY o.available_at, o.id + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE outbox_messages o + SET claimed_by = $2, + claimed_until = now() + $3::interval + FROM due + WHERE o.id = due.id + RETURNING o.id, o.job_id, o.attempt, o.kind, o.routing_key, o.payload, + o.available_at, o.published_at, COALESCE(o.claimed_by, ''), + o.claimed_until, o.publish_attempts, o.last_error, o.created_at`, + limit, owner, intervalString(claimTTL)) + if err != nil { + return nil, fmt.Errorf("claim outbox: %w", mapError(err)) + } + defer rows.Close() + + messages := make([]domain.OutboxMessage, 0, limit) + + for rows.Next() { + var ( + message domain.OutboxMessage + claimEnd pgtype.Timestamptz + errCode pgtype.Text + pubAt pgtype.Timestamptz + ) + + if err := rows.Scan( + &message.ID, &message.JobID, &message.Attempt, &message.Kind, + &message.RoutingKey, &message.Payload, &message.AvailableAt, &pubAt, + &message.ClaimedBy, &claimEnd, &message.PublishAttempts, &errCode, + &message.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("scan outbox: %w", mapError(err)) + } + + if pubAt.Valid { + value := pubAt.Time.UTC() + message.PublishedAt = &value + } + if claimEnd.Valid { + value := claimEnd.Time.UTC() + message.ClaimedUntil = &value + } + if errCode.Valid { + value := errCode.String + message.LastError = &value + } + + messages = append(messages, message) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("claim outbox: %w", mapError(err)) + } + + return messages, nil +} + +func (r *PipelineRepository) MarkPublished(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + UPDATE outbox_messages + SET published_at = now(), + claimed_by = NULL, + claimed_until = NULL + WHERE id = ANY($1::bigint[])`, ids) + if err != nil { + return fmt.Errorf("mark outbox published: %w", mapError(err)) + } + + return nil +} + +func (r *PipelineRepository) ReleaseOutbox(ctx context.Context, id int64, reason string) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + UPDATE outbox_messages + SET claimed_by = NULL, + claimed_until = NULL, + publish_attempts = publish_attempts + 1, + last_error = $2, + available_at = now() + interval '5 seconds' + WHERE id = $1`, id, reason) + if err != nil { + return fmt.Errorf("release outbox: %w", mapError(err)) + } + + return nil +} + +func (r *PipelineRepository) ReapOutboxClaims(ctx context.Context) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + UPDATE outbox_messages + SET claimed_by = NULL, claimed_until = NULL + WHERE published_at IS NULL AND claimed_until < now()`) + if err != nil { + return fmt.Errorf("reap outbox claims: %w", mapError(err)) + } + + return nil +} + +func (r *PipelineRepository) PruneOutbox(ctx context.Context, retention time.Duration) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + DELETE FROM outbox_messages + WHERE published_at IS NOT NULL AND published_at < now() - $1::interval`, + intervalString(retention)) + if err != nil { + return fmt.Errorf("prune outbox: %w", mapError(err)) + } + + return nil +} + +func (r *PipelineRepository) OldestUnpublished(ctx context.Context) (time.Time, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + var oldest pgtype.Timestamptz + + err := r.store.pool.QueryRow(ctx, ` + SELECT min(created_at) FROM outbox_messages WHERE published_at IS NULL`).Scan(&oldest) + if err != nil { + return time.Time{}, fmt.Errorf("oldest unpublished: %w", mapError(err)) + } + + if !oldest.Valid { + return time.Time{}, nil + } + + return oldest.Time.UTC(), nil +} + +// Reprocess enqueues a new pipeline version without publishing it until the +// worker commits all variants. The serving version is switched atomically by +// CompleteJob. +func (r *PipelineRepository) Reprocess( + ctx context.Context, + attachmentID uuid.UUID, + targetVersion, maxAttempts int, + jobID uuid.UUID, +) (domain.Job, error) { + var created domain.Job + + err := r.withTx(ctx, func(tx pgx.Tx) error { + // Keep the global lock order job -> attachment, as in CompleteJob. + if _, err := tx.Exec(ctx, ` + SELECT 1 FROM attachment_jobs + WHERE attachment_id = $1 AND status IN ('queued', 'processing') + AND pipeline_version < $2 + FOR UPDATE`, attachmentID, targetVersion); err != nil { + return fmt.Errorf("lock superseded jobs: %w", mapError(err)) + } + + attachmentRow := tx.QueryRow(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments WHERE id = $1 FOR UPDATE`, attachmentID) + + attachment, err := scanAttachmentRow(attachmentRow) + if err != nil { + return fmt.Errorf("load reprocess attachment: %w", mapError(err)) + } + + if targetVersion <= attachment.PipelineVersion { + return fmt.Errorf("%w: target pipeline version must be newer", domain.ErrConflict) + } + + if attachment.Status == domain.AttachmentPending { + return fmt.Errorf("%w: attachment upload is not complete", domain.ErrConflict) + } + + // The verified upload lives in staging until the first pipeline + // publishes the final original. A queued/processing/failed attachment + // must reprocess from staging: object_key points at a final original + // that may not exist yet (worker stopped before the first run), and + // choosing it would supersede v1 and leave the attachment queued with + // no jobs after v2 fails with source_missing. + sourceKey := attachment.StagingKey + if attachment.Status == domain.AttachmentReady { + // Ready means the final original was published; staging may have + // passed its retention window already. + sourceKey = attachment.ObjectKey + } + if sourceKey == "" { + sourceKey = attachment.ObjectKey + } + if sourceKey == "" { + return fmt.Errorf("%w: attachment has no source object", domain.ErrConflict) + } + + rows, err := tx.Query(ctx, ` + UPDATE attachment_jobs + SET status = 'superseded', updated_at = now(), completed_at = now() + WHERE attachment_id = $1 AND status IN ('queued', 'processing') + AND pipeline_version < $2 + RETURNING job_id, attempt`, attachmentID, targetVersion) + if err != nil { + return fmt.Errorf("supersede old jobs: %w", mapError(err)) + } + + type supersededJob struct { + id uuid.UUID + attempt int + } + + superseded := make([]supersededJob, 0, 2) + + for rows.Next() { + var old supersededJob + if err := rows.Scan(&old.id, &old.attempt); err != nil { + rows.Close() + + return fmt.Errorf("scan superseded job: %w", mapError(err)) + } + + superseded = append(superseded, old) + } + + if err := rows.Err(); err != nil { + rows.Close() + + return fmt.Errorf("list superseded jobs: %w", mapError(err)) + } + + rows.Close() + + for _, old := range superseded { + if err := insertEventTx(ctx, tx, old.id, domain.EventSuperseded, old.attempt, ""); err != nil { + return err + } + } + + if attachment.Status != domain.AttachmentReady { + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready'`, attachmentID) + if err != nil { + return fmt.Errorf("mark reprocess attachment queued: %w", mapError(err)) + } + } + + job := domain.Job{ + ID: jobID, + AttachmentID: attachmentID, + PipelineVersion: targetVersion, + SourceKey: sourceKey, + Attempt: 1, + MaxAttempts: maxAttempts, + CreatedAt: time.Now().UTC(), + } + + _, err = tx.Exec(ctx, ` + INSERT INTO attachment_jobs ( + job_id, attachment_id, pipeline_version, source_key, + attempt, max_attempts, status, available_at + ) + VALUES ($1, $2, $3, $4, 1, $5, 'queued', now())`, + job.ID, job.AttachmentID, job.PipelineVersion, job.SourceKey, job.MaxAttempts) + if err != nil { + return fmt.Errorf("create reprocess job: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventReprocessed, 1, ""); err != nil { + return err + } + + if err := insertOutboxTx( + ctx, tx, job, 1, "reprocess", time.Now().UTC(), + ); err != nil { + return err + } + + created = job + + return nil + }) + if err != nil { + return domain.Job{}, err + } + + return created, nil +} + +func (r *PipelineRepository) ReplayJob( + ctx context.Context, + jobID uuid.UUID, + extraAttempts int, +) (domain.Job, error) { + if extraAttempts < 1 { + return domain.Job{}, fmt.Errorf("%w: replay attempts must be >= 1", domain.ErrValidation) + } + + var replayed domain.Job + + err := r.withTx(ctx, func(tx pgx.Tx) error { + job, attachment, err := lockJobAndAttachment(ctx, tx, jobID) + if err != nil { + return err + } + + if job.Status != domain.JobFailed { + return fmt.Errorf("%w: only failed jobs can be replayed", domain.ErrConflict) + } + + if job.PipelineVersion < attachment.PipelineVersion { + return fmt.Errorf("%w: job is superseded by a newer pipeline version", domain.ErrConflict) + } + + // A newer version may still be unpublished (attachment.pipeline_version + // changes only when it commits), so compare against job rows too. An + // older replay must not compete with a newer job for the same staging + // source or return the attachment to an older state. + var newerJob bool + + if err := tx.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM attachment_jobs + WHERE attachment_id = $1 AND pipeline_version > $2 + )`, job.AttachmentID, job.PipelineVersion).Scan(&newerJob); err != nil { + return fmt.Errorf("check newer pipeline version: %w", mapError(err)) + } + + if newerJob { + return fmt.Errorf("%w: a newer pipeline version already has a job", domain.ErrConflict) + } + + nextAttempt := job.Attempt + 1 + + _, err = tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'queued', + attempt = $2, + max_attempts = $2::int + $3::int - 1, + available_at = now(), + lease_owner = NULL, + lease_expires_at = NULL, + last_error_code = NULL, + replay_count = replay_count + 1, + completed_at = NULL, + updated_at = now() + WHERE job_id = $1`, job.ID, nextAttempt, extraAttempts) + if err != nil { + return fmt.Errorf("replay job: %w", mapError(err)) + } + + // Mirror failJobTx: a replay of a newer unpublished version returns + // the attachment to queued so the API does not keep showing a stale + // terminal state while the replayed job waits. A published ready + // version is never disturbed by a newer version's replay. + switch { + case job.PipelineVersion == attachment.PipelineVersion: + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = now() + WHERE id = $1 AND pipeline_version = $2`, + attachment.ID, job.PipelineVersion) + case job.PipelineVersion > attachment.PipelineVersion && attachment.Status != domain.AttachmentReady: + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready'`, attachment.ID) + } + if err != nil { + return fmt.Errorf("replay attachment: %w", mapError(err)) + } + + if err := insertEventTx(ctx, tx, job.ID, domain.EventReplayed, nextAttempt, ""); err != nil { + return err + } + + job.Attempt = nextAttempt + job.MaxAttempts = nextAttempt + extraAttempts - 1 + + if err := insertOutboxTx( + ctx, tx, job, nextAttempt, "replay", time.Now().UTC(), + ); err != nil { + return err + } + + replayed = job + + return nil + }) + if err != nil { + return domain.Job{}, err + } + + return replayed, nil +} + +// Duplicate inserts another publication of the current job attempt. It exists +// only for deterministic duplicate-delivery evidence. +func (r *PipelineRepository) Duplicate(ctx context.Context, attachmentID uuid.UUID) (domain.OutboxMessage, error) { + var created domain.OutboxMessage + + err := r.withTx(ctx, func(tx pgx.Tx) error { + jobRow := tx.QueryRow(ctx, ` + SELECT `+jobReturning+` + FROM attachment_jobs + WHERE attachment_id = $1 + ORDER BY pipeline_version DESC + LIMIT 1`, attachmentID) + + job, err := scanJobRow(jobRow) + if err != nil { + return fmt.Errorf("load duplicate job: %w", mapError(err)) + } + + if err := insertOutboxTx( + ctx, tx, job, job.Attempt, "duplicate", time.Now().UTC(), + ); err != nil { + return err + } + + created = domain.OutboxMessage{ + JobID: job.ID, + Attempt: job.Attempt, + Kind: "duplicate", + RoutingKey: defaultRoutingKey, + } + + return nil + }) + if err != nil { + return domain.OutboxMessage{}, err + } + + return created, nil +} + +func (r *PipelineRepository) ListFailedJobs( + ctx context.Context, + limit int, +) ([]domain.Job, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + SELECT `+jobReturning+` + FROM attachment_jobs + WHERE status = 'failed' + ORDER BY updated_at DESC + LIMIT $1`, limit) + if err != nil { + return nil, fmt.Errorf("list failed jobs: %w", mapError(err)) + } + defer rows.Close() + + jobs := make([]domain.Job, 0, limit) + + for rows.Next() { + job, err := scanJobRow(rows) + if err != nil { + return nil, fmt.Errorf("scan failed job: %w", mapError(err)) + } + jobs = append(jobs, job) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list failed jobs: %w", mapError(err)) + } + + return jobs, nil +} + +func (r *PipelineRepository) GetJob(ctx context.Context, jobID uuid.UUID) (domain.Job, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + row := r.store.pool.QueryRow(ctx, ` + SELECT `+jobReturning+` FROM attachment_jobs WHERE job_id = $1`, jobID) + + job, err := scanJobRow(row) + if err != nil { + return domain.Job{}, fmt.Errorf("get job: %w", mapError(err)) + } + + return job, nil +} + +func (r *PipelineRepository) GetJobByAttachment( + ctx context.Context, + attachmentID uuid.UUID, + pipelineVersion int, +) (domain.Job, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + row := r.store.pool.QueryRow(ctx, ` + SELECT `+jobReturning+` + FROM attachment_jobs + WHERE attachment_id = $1 AND pipeline_version = $2`, + attachmentID, pipelineVersion) + + job, err := scanJobRow(row) + if err != nil { + return domain.Job{}, fmt.Errorf("get job by attachment: %w", mapError(err)) + } + + return job, nil +} + +func (r *PipelineRepository) GetForID(ctx context.Context, id uuid.UUID) (domain.Attachment, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + row := r.store.pool.QueryRow(ctx, ` + SELECT `+attachmentReturning+` FROM attachments a WHERE a.id = $1`, id) + + attachment, err := scanAttachmentRow(row) + if err != nil { + return domain.Attachment{}, fmt.Errorf("get attachment: %w", mapError(err)) + } + + return attachment, nil +} + +// MarkStagingDeleted clears the staging projection after the worker removed +// the object (or the sweeper verified it is gone). +func (r *PipelineRepository) MarkStagingDeleted( + ctx context.Context, + attachmentID uuid.UUID, + stagingKey string, +) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + UPDATE attachments + SET staging_key = NULL, updated_at = now() + WHERE id = $1 AND staging_key = $2`, attachmentID, stagingKey) + if err != nil { + return fmt.Errorf("clear staging key: %w", mapError(err)) + } + + return nil +} + +// ListReferencedKeys returns object keys that must never be collected: +// originals, variants, and staging of active jobs. +func (r *PipelineRepository) ListReferencedKeys( + ctx context.Context, + afterKey string, + limit int, +) ([]string, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + SELECT key FROM ( + SELECT object_key AS key FROM attachments + UNION + SELECT object_key AS key FROM attachment_variants + UNION + SELECT staging_key AS key FROM attachments + WHERE staging_key IS NOT NULL AND status IN ('queued', 'processing', 'failed') + ) referenced + WHERE key > $1 + ORDER BY key + LIMIT $2`, afterKey, limit) + if err != nil { + return nil, fmt.Errorf("list referenced keys: %w", mapError(err)) + } + defer rows.Close() + + keys := make([]string, 0, limit) + + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + return nil, fmt.Errorf("scan referenced key: %w", mapError(err)) + } + keys = append(keys, key) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list referenced keys: %w", mapError(err)) + } + + return keys, nil +} + +func (r *PipelineRepository) ListStaleStaging( + ctx context.Context, + readyBefore, failedBefore time.Time, + limit int, +) ([]domain.StaleStaging, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + SELECT id, staging_key, status + FROM attachments + WHERE staging_key IS NOT NULL + AND ( + (status = 'ready' AND updated_at < $1) OR + (status = 'failed' AND updated_at < $2) + ) + ORDER BY updated_at + LIMIT $3`, readyBefore, failedBefore, limit) + if err != nil { + return nil, fmt.Errorf("list stale staging: %w", mapError(err)) + } + defer rows.Close() + + stale := make([]domain.StaleStaging, 0, limit) + + for rows.Next() { + var ( + row domain.StaleStaging + status string + ) + if err := rows.Scan(&row.AttachmentID, &row.StagingKey, &status); err != nil { + return nil, fmt.Errorf("scan stale staging: %w", mapError(err)) + } + row.Status = domain.AttachmentStatus(status) + stale = append(stale, row) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list stale staging: %w", mapError(err)) + } + + return stale, nil +} + +func (r *PipelineRepository) ListPendingPresign( + ctx context.Context, + createdBefore time.Time, + limit int, +) ([]domain.Attachment, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + rows, err := r.store.pool.Query(ctx, ` + SELECT `+attachmentReturning+` + FROM attachments a + WHERE status = 'pending' AND created_at < $1 + ORDER BY created_at + LIMIT $2`, createdBefore, limit) + if err != nil { + return nil, fmt.Errorf("list pending presign: %w", mapError(err)) + } + defer rows.Close() + + pending := make([]domain.Attachment, 0, limit) + + for rows.Next() { + attachment, err := scanAttachmentRow(rows) + if err != nil { + return nil, fmt.Errorf("scan pending presign: %w", mapError(err)) + } + pending = append(pending, attachment) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list pending presign: %w", mapError(err)) + } + + return pending, nil +} + +func (r *PipelineRepository) DeletePending(ctx context.Context, id uuid.UUID) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id) + if err != nil { + return fmt.Errorf("delete pending attachment: %w", mapError(err)) + } + + return nil +} + +func (r *PipelineRepository) InsertRawOutbox( + ctx context.Context, + jobID uuid.UUID, + attempt int, + kind string, + payload []byte, + availableAt time.Time, +) error { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + _, err := r.store.pool.Exec(ctx, ` + INSERT INTO outbox_messages (job_id, attempt, kind, routing_key, payload, available_at) + VALUES ($1, $2, $3, $4, $5::jsonb, $6)`, + jobID, attempt, kind, defaultRoutingKey, string(payload), availableAt) + if err != nil { + return fmt.Errorf("insert raw outbox: %w", mapError(err)) + } + + return nil +} + +// CreatePresign stores the internal pending row before a direct upload. It is +// never exposed by the API. +func (r *PipelineRepository) CreatePresign( + ctx context.Context, + draft domain.AttachmentDraft, + limits domain.AdmissionLimits, +) (domain.Attachment, error) { + var attachment domain.Attachment + + err := r.withTx(ctx, func(tx pgx.Tx) error { + if err := checkAdmissionTx(ctx, tx, draft.OwnerID, limits); err != nil { + return err + } + + row := tx.QueryRow(ctx, ` + INSERT INTO attachments ( + id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, status, pipeline_version, staging_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', $9, $10) + RETURNING `+attachmentReturning, + draft.ID, draft.ChatID, draft.OwnerID, draft.ObjectKey, draft.Filename, + draft.ContentType, draft.SizeBytes, draft.SHA256, + draft.PipelineVersion, draft.StagingKey, + ) + + created, err := scanAttachmentRow(row) + if err != nil { + return fmt.Errorf("create presign attachment: %w", mapError(err)) + } + + attachment = created + + return nil + }) + if err != nil { + return domain.Attachment{}, err + } + + return attachment, nil +} + +const admissionLockKey int64 = 0x41444d31 // "ADM1" + +// checkAdmissionTx serializes upload admission across API replicas with a +// transaction-scoped advisory lock, then re-counts pending/queued/processing +// rows. This closes the check-then-act window before the row is inserted. +func checkAdmissionTx(ctx context.Context, tx pgx.Tx, ownerID uuid.UUID, limits domain.AdmissionLimits) error { + if limits.PerUser == 0 && limits.Total == 0 && limits.Bytes == 0 { + return nil + } + + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, admissionLockKey); err != nil { + return fmt.Errorf("lock admission: %w", mapError(err)) + } + + var ( + total int + bytes int64 + ) + + err := tx.QueryRow(ctx, ` + SELECT count(*)::int, COALESCE(sum(size_bytes), 0)::bigint + FROM attachments + WHERE status IN ('pending', 'queued', 'processing')`).Scan(&total, &bytes) + if err != nil { + return fmt.Errorf("admission count: %w", mapError(err)) + } + + perUser := 0 + if limits.PerUser > 0 { + if err := tx.QueryRow(ctx, ` + SELECT count(*)::int FROM attachments + WHERE status IN ('pending', 'queued', 'processing') + AND owner_id = $1`, ownerID).Scan(&perUser); err != nil { + return fmt.Errorf("admission per-user count: %w", mapError(err)) + } + } + + if (limits.PerUser > 0 && perUser >= limits.PerUser) || + (limits.Total > 0 && total >= limits.Total) || + (limits.Bytes > 0 && bytes >= limits.Bytes) { + return fmt.Errorf("%w: admission limit reached", domain.ErrRateLimited) + } + + return nil +} + +// DuplicateJob publishes another copy of a specific job attempt. It is used by +// the fault-injection evidence to simulate duplicate delivery of an old version. +func (r *PipelineRepository) DuplicateJob(ctx context.Context, jobID uuid.UUID) (domain.OutboxMessage, error) { + var created domain.OutboxMessage + + err := r.withTx(ctx, func(tx pgx.Tx) error { + jobRow := tx.QueryRow(ctx, ` + SELECT `+jobReturning+` FROM attachment_jobs WHERE job_id = $1`, jobID) + + job, err := scanJobRow(jobRow) + if err != nil { + return fmt.Errorf("load duplicate job: %w", mapError(err)) + } + + if err := insertOutboxTx( + ctx, tx, job, job.Attempt, "duplicate", time.Now().UTC(), + ); err != nil { + return err + } + + created = domain.OutboxMessage{ + JobID: job.ID, Attempt: job.Attempt, Kind: "duplicate", RoutingKey: defaultRoutingKey, + } + + return nil + }) + if err != nil { + return domain.OutboxMessage{}, err + } + + return created, nil +} + +// ReconcileQueuedJobs republishes jobs whose only broker delivery was lost +// (dead-lettered, dropped, or a broker data loss). The worker CAS makes an +// extra delivery a no-op. Only one unpublished outbox row is created per job. +func (r *PipelineRepository) ReconcileQueuedJobs( + ctx context.Context, + olderThan time.Duration, + limit int, +) (int, error) { + ctx, cancel := r.store.timeout(ctx) + defer cancel() + + tag, err := r.store.pool.Exec(ctx, ` + WITH lost AS ( + SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.created_at + FROM attachment_jobs j + WHERE j.status = 'queued' + AND j.available_at < now() - $1::interval + AND NOT EXISTS ( + SELECT 1 FROM outbox_messages o + WHERE o.job_id = j.job_id AND o.published_at IS NULL + ) + ORDER BY j.available_at + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + INSERT INTO outbox_messages (job_id, attempt, kind, routing_key, payload, available_at) + SELECT job_id, attempt, 'reconcile', $3, + jsonb_build_object( + 'schema_version', 1, + 'job_id', job_id, + 'type', $3::text, + 'attachment_id', attachment_id, + 'pipeline_version', pipeline_version, + 'attempt', attempt, + 'created_at', created_at), + now() + FROM lost`, + intervalString(olderThan), limit, defaultRoutingKey) + if err != nil { + return 0, fmt.Errorf("reconcile queued jobs: %w", mapError(err)) + } + + return int(tag.RowsAffected()), nil +} diff --git a/internal/storage/postgres/pipeline_repository_test.go b/internal/storage/postgres/pipeline_repository_test.go new file mode 100644 index 00000000..59994d95 --- /dev/null +++ b/internal/storage/postgres/pipeline_repository_test.go @@ -0,0 +1,552 @@ +//go:build integration + +package postgres_test + +import ( + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + + "mtusi-messenger/internal/domain" +) + +type PipelineSuite struct { + BaseSuite +} + +func TestPipelineSuite(t *testing.T) { + suite.Run(t, new(PipelineSuite)) +} + +func (s *PipelineSuite) attachmentDraft(chat domain.Chat, owner domain.User, version int) (domain.AttachmentDraft, domain.Job) { + s.T().Helper() + + id, err := uuid.NewV7() + s.Require().NoError(err) + jobID, err := uuid.NewV7() + s.Require().NoError(err) + + draft := domain.AttachmentDraft{ + ID: id, + ChatID: chat.ID, + OwnerID: owner.ID, + ObjectKey: domain.OriginalObjectKey(chat.ID, id), + StagingKey: domain.StagingUploadKey(chat.ID, id), + Filename: "photo.png", + ContentType: "image/png", + SizeBytes: 10, + SHA256: strings.Repeat("a", 64), + Status: domain.AttachmentQueued, + PipelineVersion: version, + } + + job := domain.Job{ + ID: jobID, + AttachmentID: id, + PipelineVersion: version, + SourceKey: draft.StagingKey, + Attempt: 1, + MaxAttempts: 5, + CreatedAt: time.Now().UTC(), + } + + return draft, job +} + +func (s *PipelineSuite) TestCreateClaimCompleteAndIdempotentRedelivery() { + alice := s.createUser("alice") + bob := s.createUser("bob") + chat := s.createChat(alice, bob) + + draft, job := s.attachmentDraft(chat, alice, 1) + repo := s.store.Pipeline() + + created, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + s.Equal(domain.AttachmentQueued, created.Status) + s.Equal(1, created.PipelineVersion) + + message := domain.NewJobMessage(job) + claim, err := repo.ClaimJob(s.T().Context(), message, "lease-1", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + s.Equal(domain.AttachmentProcessing, claim.Attachment.Status) + + duplicate, err := repo.ClaimJob(s.T().Context(), message, "lease-2", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimBusyLease, duplicate.Outcome) + + variants := []domain.Variant{ + { + Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11, + Width: 256, Height: 160, SHA256: strings.Repeat("b", 64), + ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail), + }, + } + + ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants) + s.Require().NoError(err) + s.True(ok) + + ready, err := repo.GetForMember(s.T().Context(), bob.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentReady, ready.Status) + + stored, err := repo.ListVariants(s.T().Context(), draft.ID, 1) + s.Require().NoError(err) + s.Require().Len(stored, 1) + s.Equal(domain.VariantThumbnail, stored[0].Name) + + // A redelivery after ready is a no-op, even with the same attempt. + again, err := repo.ClaimJob(s.T().Context(), message, "lease-3", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAlreadyReady, again.Outcome) +} + +func (s *PipelineSuite) TestLeaseExpiryIsBoundedByMaxAttempts() { + alice := s.createUser("alice") + chat := s.createChat(alice) + + draft, job := s.attachmentDraft(chat, alice, 1) + job.MaxAttempts = 1 + + repo := s.store.Pipeline() + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease", time.Millisecond) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + time.Sleep(10 * time.Millisecond) + + reaped, err := repo.ReapExpiredLeases(s.T().Context(), 10) + s.Require().NoError(err) + s.Equal(1, reaped) + + dead, err := repo.GetJob(s.T().Context(), job.ID) + s.Require().NoError(err) + s.Equal(domain.JobFailed, dead.Status) + s.NotNil(dead.LastErrorCode) + s.Equal("retry_exhausted", *dead.LastErrorCode) +} + +func (s *PipelineSuite) TestAdmissionCountsPendingPresignAndIsAtomic() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, _ := s.attachmentDraft(chat, alice, 1) + draft.ObjectKey = "staging/presign/" + chat.ID.String() + "/" + draft.ID.String() + draft.StagingKey = draft.ObjectKey + draft.Status = domain.AttachmentPending + + limits := domain.AdmissionLimits{PerUser: 1, Total: 10} + + _, err := repo.CreatePresign(s.T().Context(), draft, limits) + s.Require().NoError(err) + + second, _ := s.attachmentDraft(chat, alice, 1) + second.ObjectKey = "staging/presign/" + chat.ID.String() + "/" + second.ID.String() + second.StagingKey = second.ObjectKey + second.Status = domain.AttachmentPending + + _, err = repo.CreatePresign(s.T().Context(), second, limits) + s.Require().ErrorIs(err, domain.ErrRateLimited) +} + +// A queued upload whose worker never ran still has its verified bytes only in +// staging. Reprocessing it must use staging as the source; pointing at the +// not-yet-written final original left the attachment queued forever after the +// v1 job was superseded (v2 then failed with source_missing). +func (s *PipelineSuite) TestReprocessQueuedUsesStagingSource() { + alice := s.createUser("alice") + bob := s.createUser("bob") + chat := s.createChat(alice, bob) + + draft, job := s.attachmentDraft(chat, alice, 1) + repo := s.store.Pipeline() + + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + created, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + s.Equal(2, created.PipelineVersion) + s.Equal(draft.StagingKey, created.SourceKey) +} + +// Once a pipeline has published the final original, the staging copy may be +// past its retention window; a reprocess of a ready attachment must source +// from the final object that actually exists. +func (s *PipelineSuite) TestReprocessReadyUsesObjectSource() { + alice := s.createUser("alice") + chat := s.createChat(alice) + + draft, job := s.attachmentDraft(chat, alice, 1) + repo := s.store.Pipeline() + + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + message := domain.NewJobMessage(job) + claim, err := repo.ClaimJob(s.T().Context(), message, "lease-1", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + variants := []domain.Variant{ + { + Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11, + Width: 256, Height: 160, SHA256: strings.Repeat("b", 64), + ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail), + }, + } + + ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants) + s.Require().NoError(err) + s.True(ok) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + created, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + s.Equal(2, created.PipelineVersion) + s.Equal(draft.ObjectKey, created.SourceKey) +} + +// A reprocess failure before any version was published must settle the +// attachment as failed instead of leaving it queued with no active jobs. +func (s *PipelineSuite) TestReprocessFailureSettlesUnpublishedAttachment() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image") + s.Require().NoError(err) + s.True(failed) + + current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentFailed, current.Status) + s.Require().NotNil(current.ErrorCode) + s.Equal("invalid_image", *current.ErrorCode) +} + +// A reprocess failure while an older ready version is serving must keep that +// version ready and downloadable; only the new job fails. +func (s *PipelineSuite) TestReprocessFailureKeepsReadyServingVersion() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-1", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + variants := []domain.Variant{ + { + Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11, + Width: 256, Height: 160, SHA256: strings.Repeat("b", 64), + ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail), + }, + } + + ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants) + s.Require().NoError(err) + s.True(ok) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + + claim, err = repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image") + s.Require().NoError(err) + s.True(failed) + + current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentReady, current.Status) + s.Equal(1, current.PipelineVersion) + s.Nil(current.ErrorCode) +} + +// Replaying a failed newer version must clear the stale terminal projection +// for an attachment that never published any version. +func (s *PipelineSuite) TestReplayNewerFailedVersionRequeuesUnpublishedAttachment() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image") + s.Require().NoError(err) + s.True(failed) + + current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentFailed, current.Status) + + replayed, err := repo.ReplayJob(s.T().Context(), reprocess.ID, 2) + s.Require().NoError(err) + s.Equal(2, replayed.Attempt) + + current, err = repo.GetForOwner(s.T().Context(), alice.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentQueued, current.Status) + s.Nil(current.ErrorCode) + + claim, err = repo.ClaimJob(s.T().Context(), domain.NewJobMessage(replayed), "lease-3", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + variants := []domain.Variant{ + { + Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11, + Width: 256, Height: 160, SHA256: strings.Repeat("c", 64), + ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 2, domain.VariantThumbnail), + }, + } + + ok, err := repo.CompleteJob(s.T().Context(), replayed.ID, replayed.Attempt, "lease-3", variants) + s.Require().NoError(err) + s.True(ok) + + current, err = repo.GetForOwner(s.T().Context(), alice.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentReady, current.Status) + s.Equal(2, current.PipelineVersion) +} + +// Replaying a failed newer version must not disturb a published ready +// serving version; only the new job returns to queued. +func (s *PipelineSuite) TestReplayNewerFailedVersionKeepsReadyServingVersion() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-1", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + variants := []domain.Variant{ + { + Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11, + Width: 256, Height: 160, SHA256: strings.Repeat("b", 64), + ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail), + }, + } + + ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants) + s.Require().NoError(err) + s.True(ok) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + + claim, err = repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image") + s.Require().NoError(err) + s.True(failed) + + replayed, err := repo.ReplayJob(s.T().Context(), reprocess.ID, 1) + s.Require().NoError(err) + s.Equal(2, replayed.Attempt) + + current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID) + s.Require().NoError(err) + s.Equal(domain.AttachmentReady, current.Status) + s.Equal(1, current.PipelineVersion) + s.Nil(current.ErrorCode) +} + +// Replaying an older failed version is refused while a newer job exists, +// even before that newer version becomes the published serving version. +func (s *PipelineSuite) TestReplayOlderFailedVersionRefusedWhenNewerJobExists() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-1", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + failed, err := repo.FailJob(s.T().Context(), job.ID, 1, "lease-1", "invalid_image") + s.Require().NoError(err) + s.True(failed) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + _, err = repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + + _, err = repo.ReplayJob(s.T().Context(), job.ID, 1) + s.Require().ErrorIs(err, domain.ErrConflict) + + var v1Status string + s.Require().NoError(s.pool.QueryRow(s.T().Context(), + `SELECT status FROM attachment_jobs WHERE job_id = $1`, job.ID).Scan(&v1Status)) + s.Equal(string(domain.JobFailed), v1Status) +} + +// A queued job for an older version is retired on claim instead of being +// acknowledged and republished by reconciliation forever. +func (s *PipelineSuite) TestClaimRetiresVersionStaleQueuedJob() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + reprocessID, err := uuid.NewV7() + s.Require().NoError(err) + + reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + variants := []domain.Variant{ + { + Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11, + Width: 256, Height: 160, SHA256: strings.Repeat("d", 64), + ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 2, domain.VariantThumbnail), + }, + } + + ok, err := repo.CompleteJob(s.T().Context(), reprocess.ID, 1, "lease-2", variants) + s.Require().NoError(err) + s.True(ok) + + // Simulate a legacy/racing replay that left the old version queued. + _, err = s.pool.Exec(s.T().Context(), ` + UPDATE attachment_jobs + SET status = 'queued', attempt = 1, last_error_code = NULL + WHERE job_id = $1`, job.ID) + s.Require().NoError(err) + + stale, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-3", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimVersionStale, stale.Outcome) + + var status string + s.Require().NoError(s.pool.QueryRow(s.T().Context(), + `SELECT status FROM attachment_jobs WHERE job_id = $1`, job.ID).Scan(&status)) + s.Equal(string(domain.JobSuperseded), status) + + var supersededEvents int + s.Require().NoError(s.pool.QueryRow(s.T().Context(), ` + SELECT count(*) FROM attachment_job_events + WHERE job_id = $1 AND event = $2`, + job.ID, string(domain.EventSuperseded)).Scan(&supersededEvents)) + s.Equal(2, supersededEvents) +} + +func (s *PipelineSuite) TestReconcileQueuedJobsRepublishesLostIntent() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + claimed, err := repo.ClaimOutbox(s.T().Context(), "relay", time.Minute, 10) + s.Require().NoError(err) + s.Require().Len(claimed, 1) + s.Require().NoError(repo.MarkPublished(s.T().Context(), []int64{claimed[0].ID})) + + reconciled, err := repo.ReconcileQueuedJobs(s.T().Context(), 0, 10) + s.Require().NoError(err) + s.Equal(1, reconciled) + + pending, err := repo.ClaimOutbox(s.T().Context(), "relay", time.Minute, 10) + s.Require().NoError(err) + s.Require().Len(pending, 1) + s.Equal("reconcile", pending[0].Kind) + s.Equal(job.ID, pending[0].JobID) +} + +func (s *PipelineSuite) TestFailedStagingStaysReferencedUntilRetention() { + alice := s.createUser("alice") + chat := s.createChat(alice) + repo := s.store.Pipeline() + + draft, job := s.attachmentDraft(chat, alice, 1) + _, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{}) + s.Require().NoError(err) + + claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease", time.Minute) + s.Require().NoError(err) + s.Equal(domain.ClaimAcquired, claim.Outcome) + + ok, err := repo.FailJob(s.T().Context(), job.ID, 1, "lease", "invalid_image") + s.Require().NoError(err) + s.True(ok) + + keys, err := repo.ListReferencedKeys(s.T().Context(), "", 100) + s.Require().NoError(err) + s.Contains(keys, draft.StagingKey) +} diff --git a/internal/storage/postgres/queries/attachments.sql b/internal/storage/postgres/queries/attachments.sql index 830468aa..930ffd0e 100644 --- a/internal/storage/postgres/queries/attachments.sql +++ b/internal/storage/postgres/queries/attachments.sql @@ -1,56 +1,3 @@ --- name: CreateAttachment :one -INSERT INTO attachments ( - id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, status -) -VALUES ( - sqlc.arg(id), - sqlc.arg(chat_id), - sqlc.arg(owner_id), - sqlc.arg(object_key), - sqlc.arg(filename), - sqlc.arg(content_type), - sqlc.arg(size_bytes), - sqlc.arg(sha256), - 'pending' -) -RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at; - --- name: MarkAttachmentReady :one -UPDATE attachments -SET status = 'ready', - object_key = sqlc.arg(object_key), - updated_at = now() -WHERE id = sqlc.arg(id) - AND status = 'pending' -RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at; - --- name: GetAttachmentForMember :one -SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type, - a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at -FROM attachments a -JOIN chat_members cm - ON cm.chat_id = a.chat_id - AND cm.user_id = sqlc.arg(user_id) -WHERE a.id = sqlc.arg(id); - --- name: GetAttachmentForOwner :one -SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type, - a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at -FROM attachments a -JOIN chat_members cm - ON cm.chat_id = a.chat_id - AND cm.user_id = a.owner_id -WHERE a.id = sqlc.arg(id) - AND a.owner_id = sqlc.arg(owner_id); - --- name: ListAttachmentStatesForChat :many -SELECT id, status -FROM attachments -WHERE chat_id = sqlc.arg(chat_id) - AND id = ANY(sqlc.arg(ids)::uuid[]); - -- name: AttachToMessage :exec INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (sqlc.arg(message_id), sqlc.arg(attachment_id), sqlc.arg(position)); @@ -60,23 +7,3 @@ SELECT message_id, attachment_id FROM message_attachments WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[]) ORDER BY message_id, position; - --- name: DeletePendingAttachment :exec -DELETE FROM attachments -WHERE id = sqlc.arg(id) - AND status = 'pending'; - --- name: ListStalePendingAttachments :many -SELECT id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at -FROM attachments -WHERE status = 'pending' - AND created_at < sqlc.arg(created_before) -ORDER BY created_at -LIMIT sqlc.arg(row_limit); - --- name: ListAttachmentKeys :many -SELECT id, object_key, status, created_at -FROM attachments -ORDER BY id -LIMIT sqlc.arg(row_limit) OFFSET sqlc.arg(row_offset); diff --git a/internal/storage/postgres/queries/idempotency.sql b/internal/storage/postgres/queries/idempotency.sql index 14f911e7..a45857bd 100644 --- a/internal/storage/postgres/queries/idempotency.sql +++ b/internal/storage/postgres/queries/idempotency.sql @@ -38,3 +38,13 @@ WHERE actor_id = sqlc.arg(actor_id) SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = sqlc.arg(id); + +-- Read-only replay lookup used before attachment-state validation. A live key +-- returns the stored message; an expired key is ignored and may be claimed. +-- name: LookupMessageIdempotency :one +SELECT message_id, request_hash +FROM message_idempotency +WHERE actor_id = sqlc.arg(actor_id) + AND chat_id = sqlc.arg(chat_id) + AND key = sqlc.arg(key) + AND expires_at > now(); diff --git a/internal/storage/postgres/store.go b/internal/storage/postgres/store.go index f0313f78..02290838 100644 --- a/internal/storage/postgres/store.go +++ b/internal/storage/postgres/store.go @@ -17,11 +17,11 @@ type Store struct { queryTimeout time.Duration idempotencyTTL time.Duration - users *UserRepository - chats *ChatRepository - messages *MessageRepository - sessions *SessionRepository - attachments *AttachmentRepository + users *UserRepository + chats *ChatRepository + messages *MessageRepository + sessions *SessionRepository + pipeline *PipelineRepository } func New(ctx context.Context, cfg config.Postgres, idempotencyTTL time.Duration) (*Store, error) { @@ -66,7 +66,7 @@ func New(ctx context.Context, cfg config.Postgres, idempotencyTTL time.Duration) store.chats = &ChatRepository{store: store} store.messages = &MessageRepository{store: store} store.sessions = &SessionRepository{store: store} - store.attachments = &AttachmentRepository{store: store} + store.pipeline = &PipelineRepository{store: store} return store, nil } @@ -79,7 +79,7 @@ func (s *Store) Messages() *MessageRepository { return s.messages } func (s *Store) Sessions() *SessionRepository { return s.sessions } -func (s *Store) Attachments() *AttachmentRepository { return s.attachments } +func (s *Store) Pipeline() *PipelineRepository { return s.pipeline } func (s *Store) Ping(ctx context.Context) error { if err := s.pool.Ping(ctx); err != nil { diff --git a/internal/transport/rest/attachments.go b/internal/transport/rest/attachments.go index 5a72a519..e9640f21 100644 --- a/internal/transport/rest/attachments.go +++ b/internal/transport/rest/attachments.go @@ -56,11 +56,14 @@ func (h *Handler) DownloadAttachment( ctx context.Context, params oas.DownloadAttachmentParams, ) (oas.DownloadAttachmentRes, error) { - if variant, ok := params.Variant.Get(); ok && variant != oas.DownloadAttachmentVariantOriginal { - return nil, fmt.Errorf("%w: variant %q is not available", domain.ErrNotFound, variant) + variant := domain.VariantOriginal + if requested, ok := params.Variant.Get(); ok { + variant = domain.VariantName(requested) } - attachment, body, err := h.attachments.Open(ctx, caller(ctx), params.AttachmentID) + attachment, metadata, body, err := h.attachments.OpenVariant( + ctx, caller(ctx), params.AttachmentID, variant, + ) if err != nil { return nil, err } @@ -73,7 +76,7 @@ func (h *Handler) DownloadAttachment( requestID := oas.NewOptString(requestIDFromContext(ctx)) reader := closeOnEOF{ReadCloser: body} - switch attachment.ContentType { + switch metadata.ContentType { case "image/png": return &oas.DownloadAttachmentOKImagePNGHeaders{ XRequestID: requestID, @@ -84,6 +87,11 @@ func (h *Handler) DownloadAttachment( XRequestID: requestID, Response: oas.DownloadAttachmentOKImageJpeg{Data: reader}, }, nil + case "image/webp": + return &oas.DownloadAttachmentOKImageWEBPHeaders{ + XRequestID: requestID, + Response: oas.DownloadAttachmentOKImageWEBP{Data: reader}, + }, nil default: _ = body.Close() @@ -110,11 +118,11 @@ func attachmentResponse(attachment domain.Attachment) oas.Attachment { Filename: attachment.Filename, SizeBytes: int(attachment.SizeBytes), SHA256: attachment.SHA256, - Status: oas.AttachmentStatusReady, - Variants: []oas.Variant{}, + Status: attachmentStatus(attachment.Status), ErrorCode: oas.NilString{Null: true}, CreatedAt: attachment.CreatedAt, UpdatedAt: attachment.UpdatedAt, + Variants: make([]oas.Variant, 0, len(attachment.Variants)), } switch attachment.ContentType { @@ -130,9 +138,37 @@ func attachmentResponse(attachment domain.Attachment) oas.Attachment { response.ErrorCode = oas.NilString{Value: *attachment.ErrorCode} } + for _, variant := range attachment.Variants { + response.Variants = append(response.Variants, oas.Variant{ + Name: oas.VariantName(variant.Name), + ContentType: oas.VariantContentType(variant.ContentType), + SizeBytes: int(variant.SizeBytes), + Width: variant.Width, + Height: variant.Height, + SHA256: variant.SHA256, + }) + } + return response } +func attachmentStatus(status domain.AttachmentStatus) oas.AttachmentStatus { + switch status { + case domain.AttachmentQueued: + return oas.AttachmentStatusQueued + case domain.AttachmentProcessing: + return oas.AttachmentStatusProcessing + case domain.AttachmentReady: + return oas.AttachmentStatusReady + case domain.AttachmentFailed: + return oas.AttachmentStatusFailed + default: + // Pending is internal and never reaches the transport; map it to the + // closest public state rather than inventing a new enum value. + return oas.AttachmentStatusQueued + } +} + type closeOnEOF struct { io.ReadCloser } diff --git a/internal/transport/rest/errors.go b/internal/transport/rest/errors.go index 53c7cb9b..4aca2729 100644 --- a/internal/transport/rest/errors.go +++ b/internal/transport/rest/errors.go @@ -60,6 +60,8 @@ func statusForDomainError(err error) int { errors.Is(err, domain.ErrUnknownMember), errors.Is(err, domain.ErrInvalidCursor): return http.StatusBadRequest + case errors.Is(err, domain.ErrRateLimited): + return http.StatusTooManyRequests case errors.Is(err, domain.ErrTooLarge): return http.StatusRequestEntityTooLarge case errors.Is(err, domain.ErrUnsupportedMedia): diff --git a/internal/transport/rest/extensions.go b/internal/transport/rest/extensions.go index 5ff6b7d7..84484eb5 100644 --- a/internal/transport/rest/extensions.go +++ b/internal/transport/rest/extensions.go @@ -176,7 +176,7 @@ func (e *Extensions) complete(w http.ResponseWriter, r *http.Request, userID uui return err } - return writeJSON(w, http.StatusOK, attachmentResponse(attachment)) + return writeJSON(w, http.StatusAccepted, attachmentResponse(attachment)) } func decodeJSON(r *http.Request, target any) error { diff --git a/internal/transport/rest/handler.go b/internal/transport/rest/handler.go index 705d3d73..9da56f23 100644 --- a/internal/transport/rest/handler.go +++ b/internal/transport/rest/handler.go @@ -56,7 +56,11 @@ type SessionService interface { type AttachmentService interface { Upload(ctx context.Context, input attachmentService.UploadInput) (domain.Attachment, error) Get(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, error) - Open(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error) + OpenVariant( + ctx context.Context, + callerID, id uuid.UUID, + name domain.VariantName, + ) (domain.Attachment, domain.Variant, io.ReadCloser, error) Presign(ctx context.Context, request domain.PresignRequest) (domain.PresignResult, error) Complete(ctx context.Context, ownerID, id uuid.UUID, sha256 string) (domain.Attachment, error) } diff --git a/internal/transport/rest/internal/oas/oas_json_gen.go b/internal/transport/rest/internal/oas/oas_json_gen.go index 2d7b593f..6acaa49f 100644 --- a/internal/transport/rest/internal/oas/oas_json_gen.go +++ b/internal/transport/rest/internal/oas/oas_json_gen.go @@ -357,8 +357,14 @@ func (s *AttachmentStatus) Decode(d *jx.Decoder) error { } // Try to use constant string. switch AttachmentStatus(v) { + case AttachmentStatusQueued: + *s = AttachmentStatusQueued + case AttachmentStatusProcessing: + *s = AttachmentStatusProcessing case AttachmentStatusReady: *s = AttachmentStatusReady + case AttachmentStatusFailed: + *s = AttachmentStatusFailed default: *s = AttachmentStatus(v) } diff --git a/internal/transport/rest/internal/oas/oas_response_encoders_gen.go b/internal/transport/rest/internal/oas/oas_response_encoders_gen.go index 003053fe..7da16d2c 100644 --- a/internal/transport/rest/internal/oas/oas_response_encoders_gen.go +++ b/internal/transport/rest/internal/oas/oas_response_encoders_gen.go @@ -4592,7 +4592,7 @@ func encodeUploadAttachmentResponse(response UploadAttachmentRes, w http.Respons } } } - w.WriteHeader(201) + w.WriteHeader(202) e := new(jx.Encoder) response.Response.Encode(e) diff --git a/internal/transport/rest/internal/oas/oas_schemas_gen.go b/internal/transport/rest/internal/oas/oas_schemas_gen.go index 47002592..e53b75a3 100644 --- a/internal/transport/rest/internal/oas/oas_schemas_gen.go +++ b/internal/transport/rest/internal/oas/oas_schemas_gen.go @@ -230,21 +230,33 @@ func (*AttachmentHeaders) uploadAttachmentRes() {} type AttachmentStatus string const ( - AttachmentStatusReady AttachmentStatus = "ready" + AttachmentStatusQueued AttachmentStatus = "queued" + AttachmentStatusProcessing AttachmentStatus = "processing" + AttachmentStatusReady AttachmentStatus = "ready" + AttachmentStatusFailed AttachmentStatus = "failed" ) // AllValues returns all AttachmentStatus values. func (AttachmentStatus) AllValues() []AttachmentStatus { return []AttachmentStatus{ + AttachmentStatusQueued, + AttachmentStatusProcessing, AttachmentStatusReady, + AttachmentStatusFailed, } } // MarshalText implements encoding.TextMarshaler. func (s AttachmentStatus) MarshalText() ([]byte, error) { switch s { + case AttachmentStatusQueued: + return []byte(s), nil + case AttachmentStatusProcessing: + return []byte(s), nil case AttachmentStatusReady: return []byte(s), nil + case AttachmentStatusFailed: + return []byte(s), nil default: return nil, errors.Errorf("invalid value: %q", s) } @@ -253,9 +265,18 @@ func (s AttachmentStatus) MarshalText() ([]byte, error) { // UnmarshalText implements encoding.TextUnmarshaler. func (s *AttachmentStatus) UnmarshalText(data []byte) error { switch AttachmentStatus(data) { + case AttachmentStatusQueued: + *s = AttachmentStatusQueued + return nil + case AttachmentStatusProcessing: + *s = AttachmentStatusProcessing + return nil case AttachmentStatusReady: *s = AttachmentStatusReady return nil + case AttachmentStatusFailed: + *s = AttachmentStatusFailed + return nil default: return errors.Errorf("invalid value: %q", data) } @@ -818,13 +839,19 @@ func (*DownloadAttachmentServiceUnavailable) downloadAttachmentRes() {} type DownloadAttachmentVariant string const ( - DownloadAttachmentVariantOriginal DownloadAttachmentVariant = "original" + DownloadAttachmentVariantOriginal DownloadAttachmentVariant = "original" + DownloadAttachmentVariantThumbnail DownloadAttachmentVariant = "thumbnail" + DownloadAttachmentVariantCrop DownloadAttachmentVariant = "crop" + DownloadAttachmentVariantOptimized DownloadAttachmentVariant = "optimized" ) // AllValues returns all DownloadAttachmentVariant values. func (DownloadAttachmentVariant) AllValues() []DownloadAttachmentVariant { return []DownloadAttachmentVariant{ DownloadAttachmentVariantOriginal, + DownloadAttachmentVariantThumbnail, + DownloadAttachmentVariantCrop, + DownloadAttachmentVariantOptimized, } } @@ -833,6 +860,12 @@ func (s DownloadAttachmentVariant) MarshalText() ([]byte, error) { switch s { case DownloadAttachmentVariantOriginal: return []byte(s), nil + case DownloadAttachmentVariantThumbnail: + return []byte(s), nil + case DownloadAttachmentVariantCrop: + return []byte(s), nil + case DownloadAttachmentVariantOptimized: + return []byte(s), nil default: return nil, errors.Errorf("invalid value: %q", s) } @@ -844,6 +877,15 @@ func (s *DownloadAttachmentVariant) UnmarshalText(data []byte) error { case DownloadAttachmentVariantOriginal: *s = DownloadAttachmentVariantOriginal return nil + case DownloadAttachmentVariantThumbnail: + *s = DownloadAttachmentVariantThumbnail + return nil + case DownloadAttachmentVariantCrop: + *s = DownloadAttachmentVariantCrop + return nil + case DownloadAttachmentVariantOptimized: + *s = DownloadAttachmentVariantOptimized + return nil default: return errors.Errorf("invalid value: %q", data) } diff --git a/internal/transport/rest/internal/oas/oas_validators_gen.go b/internal/transport/rest/internal/oas/oas_validators_gen.go index 8c2402ed..4a372d99 100644 --- a/internal/transport/rest/internal/oas/oas_validators_gen.go +++ b/internal/transport/rest/internal/oas/oas_validators_gen.go @@ -234,8 +234,14 @@ func (s *AttachmentHeaders) Validate() error { func (s AttachmentStatus) Validate() error { switch s { + case "queued": + return nil + case "processing": + return nil case "ready": return nil + case "failed": + return nil default: return errors.Errorf("invalid value: %v", s) } @@ -1103,6 +1109,12 @@ func (s DownloadAttachmentVariant) Validate() error { switch s { case "original": return nil + case "thumbnail": + return nil + case "crop": + return nil + case "optimized": + return nil default: return errors.Errorf("invalid value: %v", s) } diff --git a/internal/transport/rest/mocks/AttachmentService.go b/internal/transport/rest/mocks/AttachmentService.go index 9177c331..76eecfb7 100644 --- a/internal/transport/rest/mocks/AttachmentService.go +++ b/internal/transport/rest/mocks/AttachmentService.go @@ -200,54 +200,61 @@ func (_c *MockAttachmentService_Get_Call) RunAndReturn(run func(ctx context.Cont return _c } -// Open provides a mock function for the type MockAttachmentService -func (_mock *MockAttachmentService) Open(ctx context.Context, callerID uuid.UUID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error) { - ret := _mock.Called(ctx, callerID, id) +// OpenVariant provides a mock function for the type MockAttachmentService +func (_mock *MockAttachmentService) OpenVariant(ctx context.Context, callerID uuid.UUID, id uuid.UUID, name domain.VariantName) (domain.Attachment, domain.Variant, io.ReadCloser, error) { + ret := _mock.Called(ctx, callerID, id, name) if len(ret) == 0 { - panic("no return value specified for Open") + panic("no return value specified for OpenVariant") } var r0 domain.Attachment - var r1 io.ReadCloser - var r2 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID) (domain.Attachment, io.ReadCloser, error)); ok { - return returnFunc(ctx, callerID, id) + var r1 domain.Variant + var r2 io.ReadCloser + var r3 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) (domain.Attachment, domain.Variant, io.ReadCloser, error)); ok { + return returnFunc(ctx, callerID, id, name) } - if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID) domain.Attachment); ok { - r0 = returnFunc(ctx, callerID, id) + if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) domain.Attachment); ok { + r0 = returnFunc(ctx, callerID, id, name) } else { r0 = ret.Get(0).(domain.Attachment) } - if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, uuid.UUID) io.ReadCloser); ok { - r1 = returnFunc(ctx, callerID, id) + if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) domain.Variant); ok { + r1 = returnFunc(ctx, callerID, id, name) } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(io.ReadCloser) + r1 = ret.Get(1).(domain.Variant) + } + if returnFunc, ok := ret.Get(2).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) io.ReadCloser); ok { + r2 = returnFunc(ctx, callerID, id, name) + } else { + if ret.Get(2) != nil { + r2 = ret.Get(2).(io.ReadCloser) } } - if returnFunc, ok := ret.Get(2).(func(context.Context, uuid.UUID, uuid.UUID) error); ok { - r2 = returnFunc(ctx, callerID, id) + if returnFunc, ok := ret.Get(3).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) error); ok { + r3 = returnFunc(ctx, callerID, id, name) } else { - r2 = ret.Error(2) + r3 = ret.Error(3) } - return r0, r1, r2 + return r0, r1, r2, r3 } -// MockAttachmentService_Open_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Open' -type MockAttachmentService_Open_Call struct { +// MockAttachmentService_OpenVariant_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OpenVariant' +type MockAttachmentService_OpenVariant_Call struct { *mock.Call } -// Open is a helper method to define mock.On call +// OpenVariant is a helper method to define mock.On call // - ctx context.Context // - callerID uuid.UUID // - id uuid.UUID -func (_e *MockAttachmentService_Expecter) Open(ctx any, callerID any, id any) *MockAttachmentService_Open_Call { - return &MockAttachmentService_Open_Call{Call: _e.mock.On("Open", ctx, callerID, id)} +// - name domain.VariantName +func (_e *MockAttachmentService_Expecter) OpenVariant(ctx any, callerID any, id any, name any) *MockAttachmentService_OpenVariant_Call { + return &MockAttachmentService_OpenVariant_Call{Call: _e.mock.On("OpenVariant", ctx, callerID, id, name)} } -func (_c *MockAttachmentService_Open_Call) Run(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID)) *MockAttachmentService_Open_Call { +func (_c *MockAttachmentService_OpenVariant_Call) Run(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID, name domain.VariantName)) *MockAttachmentService_OpenVariant_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -261,21 +268,26 @@ func (_c *MockAttachmentService_Open_Call) Run(run func(ctx context.Context, cal if args[2] != nil { arg2 = args[2].(uuid.UUID) } + var arg3 domain.VariantName + if args[3] != nil { + arg3 = args[3].(domain.VariantName) + } run( arg0, arg1, arg2, + arg3, ) }) return _c } -func (_c *MockAttachmentService_Open_Call) Return(attachment domain.Attachment, readCloser io.ReadCloser, err error) *MockAttachmentService_Open_Call { - _c.Call.Return(attachment, readCloser, err) +func (_c *MockAttachmentService_OpenVariant_Call) Return(attachment domain.Attachment, variant domain.Variant, readCloser io.ReadCloser, err error) *MockAttachmentService_OpenVariant_Call { + _c.Call.Return(attachment, variant, readCloser, err) return _c } -func (_c *MockAttachmentService_Open_Call) RunAndReturn(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error)) *MockAttachmentService_Open_Call { +func (_c *MockAttachmentService_OpenVariant_Call) RunAndReturn(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID, name domain.VariantName) (domain.Attachment, domain.Variant, io.ReadCloser, error)) *MockAttachmentService_OpenVariant_Call { _c.Call.Return(run) return _c } diff --git a/internal/worker/worker.go b/internal/worker/worker.go new file mode 100644 index 00000000..67dbd057 --- /dev/null +++ b/internal/worker/worker.go @@ -0,0 +1,568 @@ +// Package worker consumes attachment jobs, enforces the lease, produces the +// contract variants and acknowledges only after the durable result is +// committed. +package worker + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "strings" + "sync" + "time" + + "github.com/google/uuid" + amqp "github.com/rabbitmq/amqp091-go" + + "mtusi-messenger/internal/broker" + "mtusi-messenger/internal/domain" + "mtusi-messenger/internal/failpoint" + "mtusi-messenger/internal/imagecheck" + "mtusi-messenger/internal/imageproc" + "mtusi-messenger/internal/platform/objectstore" +) + +type Repository interface { + ClaimJob( + ctx context.Context, + message domain.JobMessage, + leaseOwner string, + leaseTTL time.Duration, + ) (domain.ClaimResult, error) + Heartbeat(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, leaseTTL time.Duration) (bool, error) + CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, variants []domain.Variant) (bool, error) + RetryJob( + ctx context.Context, + jobID uuid.UUID, + attempt int, + leaseOwner, errorCode string, + nextAvailable time.Time, + sameAttempt bool, + kind string, + ) (failed bool, err error) + FailJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner, errorCode string) (bool, error) + MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error +} + +type Store interface { + Open(ctx context.Context, key string) (io.ReadCloser, objectstore.Object, error) + Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string, metadata map[string]string) error + Delete(ctx context.Context, key string) error +} + +type Consumer interface { + Consume(ctx context.Context, handler broker.Handler) error + Connected() bool +} + +type Config struct { + Concurrency int + LeaseTTL time.Duration + Heartbeat time.Duration + JobTimeout time.Duration + RetryBase time.Duration + RetryCap time.Duration + MaxUploadBytes int64 + MaxImagePixels int64 + Owner string +} + +type Worker struct { + repo Repository + store Store + config Config + log *slog.Logger + consumer Consumer + + mu sync.RWMutex + running bool +} + +func New(repo Repository, store Store, config Config, log *slog.Logger) *Worker { + if log == nil { + log = slog.Default() + } + + if config.Owner == "" { + config.Owner = "worker-" + uuid.NewString() + } + + return &Worker{repo: repo, store: store, config: config, log: log} +} + +// Run starts Concurrency consumer channels. It returns when the context is +// cancelled or one consumer fails; the caller recreates the broker connection. +func (w *Worker) Run(ctx context.Context, consumer Consumer) error { + w.consumer = consumer + w.setRunning(true) + defer w.setRunning(false) + + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + wg sync.WaitGroup + once sync.Once + firstErr error + ) + + for range w.config.Concurrency { + wg.Add(1) + + go func() { + defer wg.Done() + + if err := consumer.Consume(runCtx, w.handle); err != nil && runCtx.Err() == nil { + once.Do(func() { + firstErr = err + cancel() + }) + } + }() + } + + wg.Wait() + + return firstErr +} + +func (w *Worker) setRunning(value bool) { + w.mu.Lock() + defer w.mu.Unlock() + + w.running = value +} + +func (w *Worker) Ready() (consumerRunning, brokerConnected bool) { + w.mu.RLock() + running := w.running + w.mu.RUnlock() + + if w.consumer == nil { + return running, false + } + + return running, w.consumer.Connected() +} + +func (w *Worker) handle(ctx context.Context, delivery amqp.Delivery) broker.Outcome { + message, err := domain.ParseJobMessage(delivery.Body) + if err != nil { + w.log.WarnContext(ctx, "invalid job payload moved to DLQ", slog.Any("error", err)) + + return broker.Drop + } + + result, err := w.repo.ClaimJob(ctx, message, w.config.Owner, w.config.LeaseTTL) + if err != nil { + w.log.WarnContext(ctx, "claim job failed", slog.String("job_id", message.JobID.String()), slog.Any("error", err)) + + return broker.Requeue + } + + switch result.Outcome { + case domain.ClaimAcquired: + w.log.InfoContext(ctx, "job claimed", + slog.String("job_id", message.JobID.String()), + slog.String("attachment_id", message.AttachmentID.String()), + slog.Int("attempt", message.Attempt), + slog.Int("pipeline_version", message.PipelineVersion), + ) + case domain.ClaimAlreadyReady, domain.ClaimAlreadyFailed, domain.ClaimSuperseded, + domain.ClaimStaleAttempt, domain.ClaimFutureAttempt, domain.ClaimBusyLease, + domain.ClaimNotDue, domain.ClaimVersionStale, domain.ClaimJobNotFound: + w.log.DebugContext(ctx, "job delivery is a no-op", + slog.String("job_id", message.JobID.String()), + slog.String("outcome", string(result.Outcome)), + ) + + return broker.Ack + default: + return broker.Ack + } + + return w.process(ctx, result) +} + +func (w *Worker) process( + ctx context.Context, + result domain.ClaimResult, +) broker.Outcome { + job := result.Job + attachment := result.Attachment + + failpoint.Check("WORKER_AFTER_CLAIM", job.ID) + + switch failpoint.Mode("WORKER_FAIL", job.ID) { + case "permanent": + return w.fail(ctx, job, permanentError{code: "injected_failure", err: errors.New("injected permanent worker failure")}) + case "transient": + return w.fail(ctx, job, errors.New("injected transient worker failure")) + } + + jobCtx, cancel := context.WithTimeout(ctx, w.config.JobTimeout) + defer cancel() + + leaseLost := false + heartbeatDone := make(chan struct{}) + + go func() { + defer close(heartbeatDone) + + ticker := time.NewTicker(w.config.Heartbeat) + defer ticker.Stop() + + for { + select { + case <-jobCtx.Done(): + return + case <-ticker.C: + ok, err := w.repo.Heartbeat(jobCtx, job.ID, job.Attempt, w.config.Owner, w.config.LeaseTTL) + if err != nil { + // A DB blip is not proof that the lease is gone. Keep trying; + // the fencing CAS still prevents a stale commit. + w.log.WarnContext(jobCtx, "heartbeat failed", slog.Any("error", err)) + + continue + } + if !ok { + leaseLost = true + cancel() + + return + } + } + } + }() + + // The hook runs after the heartbeat starts: it can be paused to test a + // genuinely stuck (not merely slow) worker without consuming every attempt. + failpoint.Sleep("WORKER_SLEEP_AFTER_CLAIM", job.ID) + + watchdog := time.AfterFunc(w.config.JobTimeout+30*time.Second, func() { + w.log.ErrorContext(ctx, "worker job watchdog fired", + slog.String("job_id", job.ID.String()), + slog.String("attachment_id", job.AttachmentID.String()), + ) + os.Exit(70) + }) + defer watchdog.Stop() + + err := w.runJob(jobCtx, job, attachment) + cancel() + <-heartbeatDone + + if leaseLost { + w.log.WarnContext(ctx, "lease lost; reaper will requeue", + slog.String("job_id", job.ID.String()), + ) + + return broker.Ack + } + + if err == nil { + failpoint.Check("WORKER_AFTER_COMMIT", job.ID) + + return broker.Ack + } + + return w.fail(ctx, job, err) +} + +func (w *Worker) runJob( + ctx context.Context, + job domain.Job, + attachment domain.Attachment, +) error { + params, ok := imageproc.Pipeline(job.PipelineVersion) + if !ok { + return permanentError{code: "unsupported_pipeline", err: fmt.Errorf("pipeline %d", job.PipelineVersion)} + } + + sourceKey := job.SourceKey + if sourceKey == "" { + sourceKey = attachment.StagingKey + } + if sourceKey == "" { + sourceKey = attachment.ObjectKey + } + + file, size, sum, err := w.spoolSource(ctx, sourceKey) + if err != nil { + return err + } + + defer func() { _ = file.Close() }() + defer func() { _ = os.Remove(file.Name()) }() + + if size != attachment.SizeBytes || sum != attachment.SHA256 { + return permanentError{code: "source_corrupt", err: errors.New("source bytes do not match metadata")} + } + + if _, err := file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind source: %w", err) + } + + if _, err := imagecheck.Validate(file, w.config.MaxImagePixels); err != nil { + return permanentError{code: "invalid_image", err: err} + } + + if _, err := file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind source: %w", err) + } + + img, _, err := imageproc.Load(file.Name(), w.config.MaxImagePixels) + if err != nil { + return permanentError{code: "invalid_image", err: err} + } + + if sourceKey != attachment.ObjectKey { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind original: %w", err) + } + + if err := w.store.Put( + ctx, attachment.ObjectKey, file, size, attachment.ContentType, + map[string]string{"sha256": sum}, + ); err != nil { + return fmt.Errorf("publish original: %w", err) + } + } + + variants := make([]domain.Variant, 0, 3) + + generated := []func() (imageproc.Image, error){ + func() (imageproc.Image, error) { return imageproc.Thumbnail(img, params) }, + func() (imageproc.Image, error) { return imageproc.Crop(img, params) }, + func() (imageproc.Image, error) { return imageproc.Optimized(img, params) }, + } + + for idx, generate := range generated { + variantImage, err := generate() + if err != nil { + return permanentError{code: "processing_failed", err: err} + } + + key := domain.VariantObjectKey(attachment.ChatID, attachment.ID, job.PipelineVersion, variantImage.Name) + + if err := w.store.Put( + ctx, key, bytes.NewReader(variantImage.Data), variantImage.SizeBytes, + variantImage.ContentType, map[string]string{"sha256": variantImage.SHA256}, + ); err != nil { + return fmt.Errorf("publish variant %s: %w", variantImage.Name, err) + } + + if idx == 0 { + failpoint.Check("WORKER_AFTER_FIRST_VARIANT", job.ID) + } + + variants = append(variants, domain.Variant{ + Name: variantImage.Name, + ContentType: variantImage.ContentType, + SizeBytes: variantImage.SizeBytes, + Width: variantImage.Width, + Height: variantImage.Height, + SHA256: variantImage.SHA256, + ObjectKey: key, + }) + } + + failpoint.Check("WORKER_AFTER_PUT", job.ID) + + completed, err := w.repo.CompleteJob(ctx, job.ID, job.Attempt, w.config.Owner, variants) + if err != nil { + return err + } + + if !completed { + w.log.WarnContext(ctx, "completion CAS lost", slog.String("job_id", job.ID.String())) + + return nil + } + + w.log.InfoContext(ctx, "attachment ready", + slog.String("job_id", job.ID.String()), + slog.String("attachment_id", attachment.ID.String()), + slog.Int("pipeline_version", job.PipelineVersion), + ) + + if strings.HasPrefix(sourceKey, "staging/") && sourceKey != attachment.ObjectKey { + if err := w.store.Delete(ctx, sourceKey); err != nil && !errors.Is(err, domain.ErrNotFound) { + w.log.WarnContext(ctx, "staging delete failed", slog.String("object_key", sourceKey), slog.Any("error", err)) + + return nil + } + + if sourceKey == attachment.StagingKey { + if err := w.repo.MarkStagingDeleted(ctx, attachment.ID, sourceKey); err != nil { + w.log.WarnContext(ctx, "clear staging projection failed", slog.Any("error", err)) + } + } + } + + return nil +} + +func (w *Worker) spoolSource(ctx context.Context, key string) (*os.File, int64, string, error) { + body, _, err := w.store.Open(ctx, key) + if err != nil { + if errors.Is(err, domain.ErrNotFound) { + return nil, 0, "", permanentError{code: "source_missing", err: err} + } + + return nil, 0, "", fmt.Errorf("open source: %w", err) + } + defer body.Close() + + file, err := os.CreateTemp("", "lab5-worker-*") + if err != nil { + return nil, 0, "", fmt.Errorf("create worker spool: %w", err) + } + + cleanup := func() { + _ = file.Close() + _ = os.Remove(file.Name()) + } + + hasher := sha256.New() + limited := io.LimitReader(body, w.config.MaxUploadBytes+1) + buffer := make([]byte, 64<<10) + + size, err := io.CopyBuffer(io.MultiWriter(file, hasher), limited, buffer) + if err != nil { + cleanup() + + return nil, 0, "", fmt.Errorf("copy source: %w", err) + } + + if size > w.config.MaxUploadBytes { + cleanup() + + return nil, 0, "", permanentError{code: "source_too_large", err: fmt.Errorf("source exceeds %d bytes", w.config.MaxUploadBytes)} + } + + return file, size, hex.EncodeToString(hasher.Sum(nil)), nil +} + +func (w *Worker) fail(ctx context.Context, job domain.Job, err error) broker.Outcome { + if ctx.Err() != nil { + // Parent cancellation means graceful shutdown. Keep the attempt number: + // an operator restart must not consume the processing budget. The job + // context is already cancelled, so use a short detached context for the + // durable requeue. + retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + + failed, retryErr := w.repo.RetryJob( + retryCtx, job.ID, job.Attempt, w.config.Owner, "shutdown", + time.Now().Add(time.Second), true, "shutdown", + ) + if retryErr != nil { + return broker.Requeue + } + + _ = failed + + return broker.Ack + } + + code := "processing_failed" + permanent := false + + var typed permanentError + if errors.As(err, &typed) { + code = typed.code + permanent = true + } + + if errors.Is(err, domain.ErrValidation) || errors.Is(err, domain.ErrUnsupportedMedia) || + errors.Is(err, domain.ErrTooLarge) || errors.Is(err, domain.ErrConflict) { + permanent = true + if code == "processing_failed" { + code = "invalid_image" + } + } + + if errors.Is(err, context.DeadlineExceeded) { + code = "processing_timeout" + permanent = false + } + + if permanent { + w.log.WarnContext(ctx, "job permanently failed", + slog.String("job_id", job.ID.String()), + slog.String("error_code", code), + slog.Any("error", err), + ) + + ok, failErr := w.repo.FailJob(ctx, job.ID, job.Attempt, w.config.Owner, code) + if failErr != nil { + return broker.Requeue + } + if !ok { + return broker.Ack + } + + return broker.Drop + } + + next := time.Now().Add(w.backoff(job.Attempt)) + w.log.WarnContext(ctx, "job retry scheduled", + slog.String("job_id", job.ID.String()), + slog.String("error_code", code), + slog.Time("available_at", next), + slog.Any("error", err), + ) + + failed, retryErr := w.repo.RetryJob( + ctx, job.ID, job.Attempt, w.config.Owner, code, next, false, "retry", + ) + if retryErr != nil { + return broker.Requeue + } + + if failed { + return broker.Drop + } + + return broker.Ack +} + +func (w *Worker) backoff(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + + capDelay := w.config.RetryCap + base := w.config.RetryBase + + for range attempt - 1 { + base *= 2 + if base >= capDelay { + base = capDelay + + break + } + } + + if base <= 0 { + return time.Second + } + + return time.Duration(rand.Int64N(int64(base))) + time.Millisecond +} + +type permanentError struct { + code string + err error +} + +func (e permanentError) Error() string { return e.code + ": " + e.err.Error() } +func (e permanentError) Unwrap() error { return e.err } diff --git a/scripts/lab5_track2.py b/scripts/lab5_track2.py new file mode 100755 index 00000000..fc3d35b3 --- /dev/null +++ b/scripts/lab5_track2.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Track 2 evidence: versioned pipeline, duplicate delivery, stale-version guard. + +Runs N repetitions against the live stand and writes raw CSV/Markdown evidence. +This is a correctness harness, not a load test. + + python3 scripts/lab5_track2.py --reps 5 +""" +import argparse +import csv +import hashlib +import json +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +sys.path.insert(0, str(ROOT / 'student-tests')) + +from client import Client, Failure, check # noqa: E402 +from tests_helpers import compose, jobctl, psql # noqa: E402 + +FIXTURE = ROOT / 'tests/fixtures/sample.png' +EVIDENCE = ROOT / 'evidence/lab5/track2' + + +def wait_ready(client, actor, attachment, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json'] + check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code'))) + if current['status'] == 'ready': + return current + time.sleep(0.2) + raise Failure('attachment did not become ready') + + +def wait_version(attachment_id, version, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + row = psql("select pipeline_version || '|' || status from attachments where id='%s'" % attachment_id) + if row == str(version) + '|ready': + return + time.sleep(0.2) + raise Failure('attachment did not reach pipeline version %d' % version) + + +def variants_by_name(attachment): + return {v['name']: v for v in attachment['variants']} + + +def scenario_duplicate_before_effect(client, actor, chat): + compose('stop', 'worker') + try: + attached, _ = client.attachment(actor, chat) + time.sleep(1) + job = psql("select job_id from attachment_jobs where attachment_id='%s'" % attached['id']) + check(bool(job), 'job id missing') + jobctl('duplicate', '-job', job) + jobctl('duplicate', '-job', job) + finally: + compose('up', '-d', '--no-deps', 'worker', check=False) + + ready = wait_ready(client, actor, attached) + names = sorted(variants_by_name(ready)) + check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate delivery changed the variant set') + check(psql("select count(*) from attachment_variants where attachment_id='%s'" % attached['id']) == '3', + 'duplicate delivery created extra variant rows') + + return 'duplicate_before_effect', True, 'attempts=' + psql( + "select attempt from attachment_jobs where attachment_id='%s'" % attached['id']) + + +def scenario_stale_version(client, actor, chat): + attached, content = client.attachment(actor, chat) + v1 = wait_ready(client, actor, attached) + v1_variants = variants_by_name(v1) + original_sha = hashlib.sha256(content).hexdigest() + check(v1['sha256'] == original_sha, 'original checksum changed before reprocess') + + v1_job = psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=1" % attached['id']) + check(bool(v1_job), 'v1 job missing') + + reprocess = jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2') + check(reprocess.returncode == 0, 'reprocess failed: ' + reprocess.stderr) + wait_version(attached['id'], 2) + v2 = wait_ready(client, actor, attached) + v2_variants = variants_by_name(v2) + check(sorted(v2_variants) == ['crop', 'optimized', 'thumbnail'], 'v2 variant set incomplete') + check(v2_variants['thumbnail']['sha256'] != v1_variants['thumbnail']['sha256'], + 'v2 bytes must differ from v1 (pipeline parameters changed)') + check(v2['sha256'] == original_sha, 'original changed during reprocess') + before = json.dumps(v2_variants, sort_keys=True) + + # Deliver the old v1 message after v2 is published. It must be acknowledged + # without touching the serving version or its artifacts. + jobctl('duplicate', '-job', v1_job) + time.sleep(2) + + after_attachment = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=actor, schema='Attachment')['json'] + after_variants = variants_by_name(after_attachment) + check(json.dumps(after_variants, sort_keys=True) == before, 'stale v1 delivery overwrote v2 variants') + check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2', + 'serving pipeline version regressed') + check(psql("select status from attachment_jobs where job_id='%s'" % v1_job) == 'ready', + 'v1 job state changed unexpectedly') + + return 'stale_version_after_v2', True, 'v1_job=' + v1_job + + +def main(): + parser = argparse.ArgumentParser(description='Lab5 track 2 correctness harness') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--reps', type=int, default=3) + args = parser.parse_args() + + EVIDENCE.mkdir(parents=True, exist_ok=True) + rows = [] + + client = Client(args.base_url, None) + + for rep in range(1, args.reps + 1): + alice, _, _, chat = client.bootstrap() + for scenario in (scenario_duplicate_before_effect, scenario_stale_version): + name, passed, detail = scenario(client, alice, chat) + rows.append({'scenario': name, 'rep': rep, 'pass': int(passed), 'detail': detail}) + print('D%s rep=%d: %s' % (name, rep, 'PASS' if passed else 'FAIL')) + + with (EVIDENCE / 'runs.csv').open('w', newline='') as handle: + writer = csv.DictWriter(handle, fieldnames=['scenario', 'rep', 'pass', 'detail']) + writer.writeheader() + writer.writerows(rows) + + summary = [ + '# Track 2 — versioned pipeline and duplicate/stale delivery', + '', + 'Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`,', + 'deterministic object keys, a serving version that changes only when all v2', + 'variants are committed, and stale-version acknowledgements, an at-least-once', + 'pipeline loses no accepted work and never overwrites a newer result with an', + 'older duplicate.', + '', + 'Method: N=%d repetitions. Each repetition creates fresh users/chat and runs:' % args.reps, + '- duplicate pipeline publication while the worker is stopped, followed by recovery;', + '- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready.', + '', + 'Invariants checked: unique variant rows per effect key; ready remains ready;', + 'serving pipeline version never regresses; original bytes/checksum unchanged;', + 'stale job state is not rewritten.', + '', + 'Limitations: this harness covers duplicate and stale/reorder delivery. Crash', + 'windows (before/after side effect, before/after ack, outbox gap) and lease', + 'takeover are covered by student-tests/lab5_faults.py; both are single-node', + 'Compose evidence, not a production availability claim. Delivery is', + 'at-least-once; exactly-once is not claimed.', + ] + (EVIDENCE / 'summary.md').write_text('\n'.join(summary) + '\n') + + passed = sum(row['pass'] for row in rows) + print('PASS: track2 %d/%d scenarios' % (passed, len(rows))) + + +if __name__ == '__main__': + main() diff --git a/scripts/lb_readiness_check.sh b/scripts/lb_readiness_check.sh index 1b163d60..dbe40e5b 100755 --- a/scripts/lb_readiness_check.sh +++ b/scripts/lb_readiness_check.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Проверка критерия «LB исключает неготовую реплику» (lab3). +# Проверка критерия «LB исключает неготовую реплику» (lab5). # # Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её # доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и @@ -23,10 +23,10 @@ if [ -f .env ]; then set +a fi -OUT=${OUT:-evidence/lb-readiness.txt} -PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4} +OUT=${OUT:-evidence/lab5/lb-readiness-detail.txt} +PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5} NET=${PROJECT}_default -IMAGE=${IMAGE:-mtusi-messenger:lab4} +IMAGE=${IMAGE:-mtusi-messenger:lab5} STATS=${STATS:-http://127.0.0.1:8404/stats;csv} BASE_URL=${BASE_URL:-http://localhost:8080} CANDIDATE=lb-readiness-candidate @@ -77,7 +77,7 @@ wait_readiness() { } up_slots() { - curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }' + curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 ~ /^UP/ && $2 != "BACKEND" { printf "%s ", $2 }' } wait_new_up_slot() { @@ -120,7 +120,7 @@ wait_slot_status() { } { - echo "# LB excludes a not-ready but reachable replica (lab3)" + echo "# LB excludes a not-ready but reachable replica (lab5)" echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)" echo @@ -139,6 +139,11 @@ wait_slot_status() { -e POSTGRES_USER="${POSTGRES_USER:-messenger}" \ -e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \ -e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \ + -e S3_ENDPOINT=http://seaweedfs:8333 -e S3_REGION="${S3_REGION:-us-east-1}" \ + -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" \ + -e S3_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID:-lab4app}" \ + -e S3_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY:-lab4app-secret}" \ + -e S3_USE_PATH_STYLE=true \ "$IMAGE" >/dev/null if ! wait_readiness 200; then @@ -170,7 +175,8 @@ wait_slot_status() { if wait_slot_status "$slot" DOWN; then echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)" else - echo " WARNING: $slot is still not DOWN" >&2 + echo " FAIL: $slot is still not DOWN" >&2 + exit 1 fi echo " stats after failure: $(stats)" diff --git a/scripts/restart.sh b/scripts/restart.sh index 3767d829..70c3226f 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,6 +1,6 @@ #!/bin/sh set -eu -# Restart hook: bounce API replicas and every state store of the lab4 stand. +# Restart hook: bounce API replicas, worker/relay and every state store of the lab5 stand. # # scripts/restart.sh # @@ -29,16 +29,16 @@ compose() { $COMPOSE_BIN --env-file .env -f compose.yaml "$@" } -# 1. State stores: restart the running ones, then make sure they are healthy. -# Volumes are attached as before, nothing is dropped or re-seeded. -compose up -d --wait postgres redis seaweedfs -compose restart postgres redis seaweedfs -compose up -d --wait postgres redis seaweedfs +# 1. State stores and broker: restart the running ones, then make sure they are +# healthy. Volumes are attached as before, nothing is dropped or re-seeded. +compose up -d --wait postgres redis seaweedfs broker +compose restart postgres redis seaweedfs broker +compose up -d --wait postgres redis seaweedfs broker -# 2. API replicas: restart in place (same containers, same addresses) and bring -# back any replica that was stopped by the fault-injection hook. -compose restart api -compose up -d --wait --no-deps api +# 2. API replicas, worker and relay: restart in place and bring back any replica +# stopped by a fault-injection hook. The worker reconnects to the broker. +compose restart api worker relay +compose up -d --wait --no-deps api worker relay # 3. Load balancer keeps its upstream; re-resolve in case addresses changed. compose restart lb diff --git a/scripts/start-broker.sh b/scripts/start-broker.sh new file mode 100755 index 00000000..5708295b --- /dev/null +++ b/scripts/start-broker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT_DIR" +COMPOSE_BIN=${COMPOSE_BIN:-docker compose} +$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps broker diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 00000000..3c2b1707 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT_DIR" +COMPOSE_BIN=${COMPOSE_BIN:-docker compose} +$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps worker diff --git a/scripts/stop-broker.sh b/scripts/stop-broker.sh new file mode 100755 index 00000000..5aa1d0bb --- /dev/null +++ b/scripts/stop-broker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT_DIR" +COMPOSE_BIN=${COMPOSE_BIN:-docker compose} +$COMPOSE_BIN --env-file .env -f compose.yaml stop broker diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 3f3d5fb8..fbb8ede7 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -1,6 +1,6 @@ #!/bin/sh set -eu -# Fault-injection hook: stop exactly ONE api replica of the lab4 stand. +# Fault-injection hook: stop exactly ONE api replica of the lab5 stand. # # TARGET_INSTANCE_ID= scripts/stop-one.sh # @@ -13,7 +13,7 @@ set -eu # No credentials are printed. : "${TARGET_INSTANCE_ID:?test provides target API instance alias}" -PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4} +PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5} candidates=$(docker ps -q \ --filter "label=com.docker.compose.project=${PROJECT}" \ diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..26a1691e --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +# Fault-injection hook: stop the lab5 worker and leave it stopped. +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT_DIR" +COMPOSE_BIN=${COMPOSE_BIN:-docker compose} +$COMPOSE_BIN --env-file .env -f compose.yaml stop worker diff --git a/student-tests/attachment_failover.py b/student-tests/attachment_failover.py index b0138110..09d114f8 100755 --- a/student-tests/attachment_failover.py +++ b/student-tests/attachment_failover.py @@ -30,7 +30,7 @@ def main(): response = client.call( 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote('failover.png'), - 201, + 202, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, @@ -40,6 +40,7 @@ def main(): instance = response['headers'].get('X-Instance-Id') check(bool(instance), 'upload response has no X-Instance-Id') check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'upload checksum mismatch') + attachment = client.wait_ready(alice, attachment) stop = subprocess.run( [str(ROOT / 'scripts/stop-one.sh')], diff --git a/student-tests/attachments_negative.py b/student-tests/attachments_negative.py index 50b1d0a3..60330668 100755 --- a/student-tests/attachments_negative.py +++ b/student-tests/attachments_negative.py @@ -5,10 +5,12 @@ Run against a live stand: python3 student-tests/attachments_negative.py --base-url http://localhost:8080 """ import hashlib +import struct import sys import urllib.error import urllib.request import uuid +import zlib from pathlib import Path from urllib.parse import quote @@ -19,11 +21,36 @@ from client import Client, Failure, arguments, check, run # noqa: E402 FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png' -def upload(client, actor, chat, content, filename='sample.png', content_type='image/png', expected=201): +def upload(client, actor, chat, content, filename='sample.png', content_type='image/png', expected=202): path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(filename, safe='') return client.call('POST', path, expected, actor=actor, raw=content, headers={'Content-Type': content_type}) +def png_chunk(ctype, payload): + crc = zlib.crc32(ctype + payload) & 0xffffffff + return struct.pack('>I', len(payload)) + ctype + payload + struct.pack('>I', crc) + + +def crafted_png(width, height, interlace, raster, bit_depth=8, color_type=6): + """PNG with correct chunk lengths and CRCs around arbitrary IDAT bytes.""" + return crafted_png_extra( + width, height, raster, bit_depth=bit_depth, color_type=color_type, + interlace=interlace, + ) + + +def crafted_png_extra(width, height, raster, pre_idat=(), bit_depth=8, color_type=6, interlace=0): + """PNG whose IDAT is preceded by caller-supplied chunks (PLTE/tRNS/...).""" + ihdr = struct.pack('>IIBBBBB', width, height, bit_depth, color_type, 0, 0, interlace) + return ( + b'\x89PNG\r\n\x1a\n' + + png_chunk(b'IHDR', ihdr) + + b''.join(pre_idat) + + png_chunk(b'IDAT', zlib.compress(raster)) + + png_chunk(b'IEND', b'') + ) + + def main(): args = arguments('Attachment negative scenarios (lab4)') client = Client(args.base_url, args.ca_file) @@ -49,6 +76,87 @@ def main(): upload(client, alice, chat, fixture, filename='photo.jpg', content_type='image/jpeg', expected=415) print('MIME spoof (PNG declared as JPEG): 415') + # Structurally valid PNG (chunk layout, lengths, CRCs all correct) whose + # IDAT payload is not a zlib stream at all. Structure-checking alone must + # not accept it: the stream has to inflate. Regression for a review + # finding that used to produce 202 queued and then failed/invalid_image + # in the worker. + upload( + client, alice, chat, + crafted_png(64, 32, 0, b'this is not a zlib stream'), + expected=415, + ) + print('structurally valid PNG with non-zlib IDAT: 415') + + # Correct zlib stream that is not a valid PNG raster: an empty payload + # and a scanline with an undefined filter byte must both be 415, not 202. + upload(client, alice, chat, crafted_png(2, 2, 0, b''), expected=415) + print('valid zlib, empty PNG raster: 415') + invalid_filter = b'\x05' + b'\x00' * 8 + b'\x00' + b'\x00' * 8 + upload(client, alice, chat, crafted_png(2, 2, 0, invalid_filter), expected=415) + print('valid zlib, invalid PNG scanline filter: 415') + + # RGBA forbids bit depth 1; every chunk CRC is correct. + upload(client, alice, chat, crafted_png(1, 1, 0, b'\x00', bit_depth=1), expected=415) + print('invalid PNG color type/bit depth pair: 415') + + # Duplicate IHDR is forbidden even with correct CRCs. + ihdr = png_chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 6, 0, 0, 0)) + duplicate_header = ( + b'\x89PNG\r\n\x1a\n' + + ihdr + ihdr + + png_chunk(b'IDAT', zlib.compress(b'\x00' + b'\x00' * 4)) + + png_chunk(b'IEND', b'') + ) + upload(client, alice, chat, duplicate_header, expected=415) + print('duplicate PNG IHDR: 415') + + # Bytes after the zlib stream inside IDAT are not part of a valid raster. + trailing = ( + b'\x89PNG\r\n\x1a\n' + + png_chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 6, 0, 0, 0)) + + png_chunk(b'IDAT', zlib.compress(b'\x00' + b'\x00' * 4) + b'junk') + + png_chunk(b'IEND', b'') + ) + upload(client, alice, chat, trailing, expected=415) + print('trailing bytes after PNG zlib stream: 415') + + # tRNS is forbidden for alpha color types and has fixed lengths otherwise. + rgba_trns = crafted_png_extra( + 1, 1, b'\x00' + b'\x00' * 4, + pre_idat=(png_chunk(b'tRNS', b'\x00'),), color_type=6, + ) + upload(client, alice, chat, rgba_trns, expected=415) + print('tRNS on RGBA PNG: 415') + + rgb_trns = crafted_png_extra( + 1, 1, b'\x00' + b'\x00' * 3, + pre_idat=(png_chunk(b'tRNS', b'\x00'),), color_type=2, + ) + upload(client, alice, chat, rgb_trns, expected=415) + print('short tRNS on RGB PNG: 415') + + # A 1-bit palette cannot address three entries. + palette_overflow = crafted_png_extra( + 1, 1, b'\x00\x00', + pre_idat=(png_chunk(b'PLTE', bytes((255, 0, 0, 0, 255, 0, 0, 0, 255))),), + bit_depth=1, color_type=3, + ) + upload(client, alice, chat, palette_overflow, expected=415) + print('PLTE entries exceed palette bit depth: 415') + + # Palette transparency cannot describe more entries than PLTE provides. + palette_trns = crafted_png_extra( + 1, 1, b'\x00\x00', + pre_idat=( + png_chunk(b'PLTE', bytes((255, 0, 0))), + png_chunk(b'tRNS', b'\x00\x00'), + ), + bit_depth=8, color_type=3, + ) + upload(client, alice, chat, palette_trns, expected=415) + print('palette tRNS exceeds PLTE entries: 415') + # Dangerous display names. for name in ('../x.png', 'a/b.png', 'a\\b.png', 'a\x00.png', 'a\n.png'): upload(client, alice, chat, fixture, filename=name, expected=400) @@ -92,16 +200,18 @@ def main(): ) print('duplicate attachment ids: 400') - # The only variant in the lab4 contract is original; other values are 400. + # Lab5 exposes the derived variants after ready; queued content is a conflict. own, content = client.attachment(alice, chat) - client.call('GET', '/api/v1/attachments/' + own['id'] + '/content?variant=thumbnail', 400, actor=alice) + client.call('GET', '/api/v1/attachments/' + own['id'] + '/content', 409, actor=alice) + own = client.wait_ready(alice, own) + client.call('GET', '/api/v1/attachments/' + own['id'] + '/content?variant=thumbnail', 200, actor=alice) # Attachment-only message is valid. client.message(alice, chat, '', attachments=[own['id']]) print('attachment-only message: 201') # Object keys are server-generated; anonymous direct access must be denied. - key = 'attachments/' + chat['id'] + '/' + own['id'] + key = 'attachments/' + chat['id'] + '/' + own['id'] + '/original' request = urllib.request.Request('http://127.0.0.1:8333/messenger-attachments/' + key, method='GET') try: urllib.request.urlopen(request, timeout=5) diff --git a/student-tests/lab5_async.py b/student-tests/lab5_async.py new file mode 100755 index 00000000..b1735d63 --- /dev/null +++ b/student-tests/lab5_async.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Lab5: accepted work survives a stopped worker and a broker restart. + +Run against a live stand: + python3 student-tests/lab5_async.py --base-url http://localhost:8080 +""" +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from client import Client, Failure, arguments, check, run # noqa: E402 +from tests_helpers import ROOT, compose, jobctl, psql # noqa: E402 + +FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png' + + +def wait_api_ready(client, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + client.call('GET', '/health/ready', schema='Health') + return + except Failure: + time.sleep(0.2) + raise Failure('API did not become ready after restart') + + +def wait_state(client, actor, attachment, expected, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json'] + check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code'))) + if current['status'] == expected: + return current + time.sleep(0.2) + raise Failure('attachment did not reach ' + expected) + + +def main(): + args = arguments('Lab5 async worker/broker durability') + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + content = FIXTURE.read_bytes() + + # 1. Worker down: API still accepts 202, content is 409, the message links. + compose('stop', 'worker') + try: + attached, _ = client.attachment(alice, chat) + check(attached['status'] == 'queued', '202 snapshot must be queued') + queued = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json'] + check(queued['status'] == 'queued', 'worker stopped but attachment is not queued') + client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice) + client.message(alice, chat, '', attachments=[attached['id']]) + time.sleep(2) + still = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json'] + check(still['status'] == 'queued', 'worker stopped but job was processed synchronously') + # The README demo restarts the API after the 202 and before the worker. + compose('restart', 'api') + wait_api_ready(client) + print('worker down: 202 queued, content 409, queued attachment linked, API restarted') + finally: + compose('up', '-d', '--no-deps', 'worker', check=False) + + ready = client.wait_ready(alice, attached) + thumb = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=thumbnail', actor=bob) + check(thumb['headers'].get_content_type() == 'image/jpeg', 'thumbnail content type mismatch') + download = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob) + check(download['bytes'] == content, 'original bytes changed') + check(psql("select coalesce(staging_key, '') from attachments where id='%s'" % ready['id']) == '', + 'staging key was not cleared after ready') + print('worker restarted: ready, thumbnail and original downloadable, staging cleared') + + # 2. Broker restart with its volume: an intent accepted while the broker is + # down is published by the relay after recovery. + compose('stop', 'relay') + compose('stop', 'broker') + try: + during, _ = client.attachment(alice, chat) + check(during['status'] == 'queued', 'upload during broker outage must be queued') + print('broker down: upload accepted into outbox') + finally: + compose('up', '-d', '--wait', '--no-deps', 'broker', check=False) + compose('up', '-d', '--no-deps', 'relay', check=False) + + ready = client.wait_ready(alice, during, wait_seconds=90) + print('broker + relay recovered: queued job reached ready') + + # 3. A full stack restart with volumes keeps ready data and the worker path. + compose('restart', 'api', 'worker', 'relay') + wait_api_ready(client) + recovered = client.call('GET', '/api/v1/attachments/' + ready['id'], actor=bob, schema='Attachment')['json'] + check(recovered['status'] == 'ready', 'ready attachment lost across restart') + body = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob) + check(body['bytes'] == content, 'original lost across restart') + print('stack restart: ready attachment and original preserved') + + # 4. Reprocess of a queued upload while the worker is down. The final + # original has never been written, so pipeline 2 must source from staging; + # pointing at the not-yet-existing final original superseded v1 and left + # the attachment queued forever with v2 failed (source_missing). + compose('stop', 'worker') + try: + pinned, _ = client.attachment(alice, chat) + check(pinned['status'] == 'queued', 'reprocess fixture must be queued') + result = jobctl('reprocess', '-attachment', pinned['id'], '-pipeline', '2') + check('SourceKey' in result.stdout, 'reprocess did not return the created job') + queued2 = client.call('GET', '/api/v1/attachments/' + pinned['id'], actor=alice, schema='Attachment')['json'] + check(queued2['status'] == 'queued', 'reprocessed upload is not queued') + check(psql("select count(*) from attachment_job_events e join attachment_jobs j on j.job_id=e.job_id where j.attachment_id='%s' and j.pipeline_version=1 and e.event='superseded'" % pinned['id']) == '1', + 'reprocess did not record the v1 superseded event') + print('worker down: reprocess pipeline 2 accepted, v1 audit event recorded, upload still queued') + finally: + compose('up', '-d', '--no-deps', 'worker', check=False) + + ready2 = client.wait_ready(alice, pinned, wait_seconds=90) + check(psql("select pipeline_version from attachments where id='%s'" % pinned['id']) == '2', + 'serving pipeline version is not 2 after reprocess') + check(psql("select status from attachment_jobs where attachment_id='%s' order by pipeline_version" % pinned['id']) == 'superseded\nready', + 'old job not superseded or new job not ready') + print('worker up: reprocessed upload reached ready at pipeline 2, v1 superseded') + + # 5. A pipeline version the worker cannot execute must be refused by the + # operator CLI before any state change; the accepted v1 upload keeps its + # job and finishes normally instead of being superseded into queued. + compose('stop', 'worker') + try: + preserved, _ = client.attachment(alice, chat) + check(preserved['status'] == 'queued', 'unsupported-pipeline fixture must be queued') + refused = jobctl('reprocess', '-attachment', preserved['id'], '-pipeline', '3', check=False) + check(refused.returncode != 0 and 'unsupported pipeline version 3' in refused.stderr, + 'unsupported pipeline version was not rejected') + check(psql("select status from attachment_jobs where attachment_id='%s' and pipeline_version=1" % preserved['id']) == 'queued', + 'v1 job changed by rejected reprocess') + print('worker down: reprocess pipeline 3 refused, v1 job untouched') + finally: + compose('up', '-d', '--no-deps', 'worker', check=False) + + client.wait_ready(alice, preserved, wait_seconds=90) + check(psql("select pipeline_version from attachments where id='%s'" % preserved['id']) == '1', + 'rejected reprocess changed serving version') + print('worker up: preserved upload reached ready at pipeline 1') + + print('PASS: lab5 async worker/broker durability') + + +if __name__ == '__main__': + try: + run(main) + finally: + compose('up', '-d', '--no-deps', 'worker', 'relay', check=False) diff --git a/student-tests/lab5_faults.py b/student-tests/lab5_faults.py new file mode 100755 index 00000000..0df32834 --- /dev/null +++ b/student-tests/lab5_faults.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""Lab5 level 4: retry, DLQ/replay, crash windows, lease takeover. + +The stand must be running. The script temporarily recreates worker/relay with +FAULTS_ENABLED=1 and targeted failpoints, then restores the normal services. + + python3 student-tests/lab5_faults.py --base-url http://localhost:8080 +""" +import json +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from client import Client, Failure, arguments, check, run # noqa: E402 +from tests_helpers import compose, jobctl, psql # noqa: E402 + +FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png' +WORKER = 'mtusi-messenger-lab5-worker-1' +RELAY = 'mtusi-messenger-lab5-relay-1' + + +def worker_with(**env): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=env) + + +def relay_with(**env): + compose('up', '-d', '--force-recreate', '--no-deps', 'relay', env=env) + + +def reset_worker(): + worker_with() + + +def reset_relay(): + relay_with() + + +def job_id(attachment_id): + value = psql("select job_id from attachment_jobs where attachment_id = '%s'" % attachment_id) + check(bool(value), 'job row is missing for ' + attachment_id) + + return value.splitlines()[0] + + +def wait_db(attachment_id, predicate, timeout=120, description=''): + deadline = time.monotonic() + timeout + last = '' + while time.monotonic() < deadline: + rows = psql("select a.status || '|' || coalesce(a.error_code,'') || '|' || j.attempt from attachments a join attachment_jobs j on j.attachment_id=a.id where a.id='%s'" % attachment_id) + if rows: + last = rows.splitlines()[0] + status, error, attempt = last.split('|') + if predicate(status, error, int(attempt)): + return status, error, int(attempt) + time.sleep(0.2) + raise Failure('timeout waiting for ' + description + ' (last=' + last + ')') + + +def queue_messages(name): + project = 'mtusi-messenger-lab5' + result = subprocess.run( + ['docker', 'exec', project + '-broker-1', 'rabbitmqctl', '-p', 'messenger', 'list_queues', 'name', 'messages'], + capture_output=True, text=True, check=True, + ) + for line in result.stdout.splitlines(): + if line.startswith(name): + return int(line.split()[1]) + return 0 + + +def scenario_permanent_failure_and_replay(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + + worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='permanent') + status, code, attempts = wait_db(attached['id'], lambda s, e, a: s == 'failed', 40, 'permanent failure') + check(code == 'injected_failure', 'unexpected permanent error_code: ' + code) + check(attempts == 1, 'a permanent error must not consume retries') + meta = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json'] + check(meta['status'] == 'failed' and meta['error_code'] == code, 'API did not expose failed/error_code') + client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice) + client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': '', 'attachment_ids': [attached['id']]}, + headers={'Idempotency-Key': 'failed-link-' + attached['id']}, + ) + failed_view = jobctl('failed', '-limit', '20').stdout + check(jid in failed_view, 'failed job is missing from the operator registry') + print('permanent failure: failed/' + code + ', content 409, failed attachment not linkable, registry visible') + + reset_worker() + replay = jobctl('replay', '-job', jid) + check(replay.returncode == 0, 'jobctl replay failed: ' + replay.stderr) + ready = client.wait_ready(alice, attached, wait_seconds=60) + check(len([v for v in ready['variants'] if v['name'] == 'thumbnail']) == 1, 'replay created a duplicate thumbnail') + print('replay: failed -> queued -> ready, one thumbnail') + + # Replay of a ready job must be refused and must not create another effect. + again = jobctl('replay', '-job', jid, check=False) + check(again.returncode != 0, 'replay of a ready job must be refused') + print('replay ready job: refused') + + +def scenario_transient_exhaustion(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + + dlq_before = queue_messages('attachment.process.dead') + + worker_with( + FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='transient', + PROCESSING_RETRY_BASE='100ms', PROCESSING_RETRY_CAP='500ms', + ) + status, code, attempts = wait_db(attached['id'], lambda s, e, a: s == 'failed', 120, 'retry exhaustion') + check(attempts == 5, 'retry cap must be exactly 5, got ' + str(attempts)) + check(code == 'retry_exhausted', 'unexpected exhaustion code: ' + code) + events = psql("select event || ':' || coalesce(attempt,0) from attachment_job_events where job_id='%s' order by id" % jid) + check(events.count('retried:') == 4, 'expected four managed retries: ' + events) + check(events.count('failed:') == 1, 'expected one terminal failure: ' + events) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if queue_messages('attachment.process.dead') >= dlq_before + 1: + break + time.sleep(0.2) + + check(queue_messages('attachment.process.dead') >= dlq_before + 1, + 'terminal failure did not add a DLQ message') + print('transient exhaustion: attempts=5, retry_exhausted, four retries, DLQ message present') + + reset_worker() + + +def scenario_crash_after_first_variant(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + + relay_with(OUTBOX_REAPER_INTERVAL='2s') + worker_with( + FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_FIRST_VARIANT='exit', + PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='10s', + ) + + ready = client.wait_ready(alice, attached, wait_seconds=90) + status, _, attempts = wait_db(attached['id'], lambda s, e, a: s == 'ready', 10, 'crash recovery') + check(attempts >= 2, 'crash recovery must use a new managed attempt, got ' + str(attempts)) + names = sorted(v['name'] for v in ready['variants']) + check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate/partial variant set: ' + str(names)) + print('crash after first variant: redelivered, same three variants, attempts=' + str(attempts)) + + reset_worker() + reset_relay() + + +def scenario_crash_after_commit(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + + worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_COMMIT='exit') + ready = client.wait_ready(alice, attached, wait_seconds=60) + check(ready['status'] == 'ready', 'job did not stay ready after commit/ack crash') + check(psql("select attempt from attachment_jobs where job_id='%s'" % jid) == '1', + 'redelivery after ready must be a no-op, not a new attempt') + print('crash after commit before ack: ready remains, redelivery is a no-op') + + reset_worker() + + +def scenario_crash_before_side_effect(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + + relay_with(OUTBOX_REAPER_INTERVAL='2s') + worker_with( + FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_CLAIM='exit', + PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='12s', + ) + + ready = client.wait_ready(alice, attached, wait_seconds=90) + status, _, attempts = wait_db(attached['id'], lambda s, e, a: s == 'ready', 10, 'pre-effect crash recovery') + check(attempts >= 2, 'pre-effect crash must use a new managed attempt, got ' + str(attempts)) + check(sorted(v['name'] for v in ready['variants']) == ['crop', 'optimized', 'thumbnail'], + 'pre-effect crash produced an incomplete variant set') + print('crash after claim before side effect: requeued, attempts=' + str(attempts) + ', ready') + + reset_worker() + reset_relay() + + +def scenario_outbox_gap(client, alice, chat): + compose('stop', 'relay') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + unpublished = psql("select count(*) from outbox_messages where job_id='%s' and published_at is null" % jid) + check(unpublished == '1', 'upload did not leave a durable outbox intent') + + relay_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_OUTBOX_AFTER_COMMIT='exit') + ready = client.wait_ready(alice, attached, wait_seconds=90) + check(psql("select count(*) from outbox_messages where job_id='%s' and published_at is null" % jid) == '0', + 'outbox row stayed unpublished after recovery') + print('outbox gap: crash before publish recovered through the relay, outbox published, ready') + reset_relay() + + +def scenario_lease_takeover(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid = job_id(attached['id']) + + relay_with(OUTBOX_REAPER_INTERVAL='1s') + worker_with( + FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_SLEEP_AFTER_CLAIM='8s', + PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='12s', + ) + + wait_db(attached['id'], lambda s, e, a: s == 'processing', 20, 'processing claim') + subprocess.run(['docker', 'pause', WORKER], check=True, capture_output=True) + wait_db(attached['id'], lambda s, e, a: s == 'queued' and a >= 2, 30, 'lease expiry requeue') + subprocess.run(['docker', 'unpause', WORKER], check=True, capture_output=True) + ready = client.wait_ready(alice, attached, wait_seconds=60) + check(len(ready['variants']) == 3, 'lease takeover produced an incomplete variant set') + print('lease expiry: paused worker, reaper requeued, stale attempt fenced, ready') + + reset_worker() + reset_relay() + + +def scenario_replay_reprocessed_version(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + reprocessed = json.loads(jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2').stdout) + jid = reprocessed['ID'] + + worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='permanent') + + deadline = time.monotonic() + 40 + last = '' + while time.monotonic() < deadline: + job_status = psql("select status from attachment_jobs where job_id='%s'" % jid) + last = psql("select status || '|' || coalesce(error_code,'') from attachments where id='%s'" % attached['id']) + if job_status == 'failed' and last.startswith('failed|'): + break + time.sleep(0.2) + + check(job_status == 'failed' and last == 'failed|injected_failure', + 'reprocessed failure did not settle: ' + last) + + compose('stop', 'worker') + jobctl('replay', '-job', jid) + projection = client.call( + 'GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment', + )['json'] + check(projection['status'] == 'queued' and projection['error_code'] is None, + 'replay left stale terminal projection: ' + str(projection['status'])) + + reset_worker() + client.wait_ready(alice, attached, wait_seconds=90) + check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2', + 'replayed reprocess did not publish pipeline 2') + print('replay of failed reprocess: queued projection restored, then ready at pipeline 2') + + +def scenario_replay_older_version_refused(client, alice, chat): + compose('stop', 'worker') + attached, _ = client.attachment(alice, chat) + jid_v1 = job_id(attached['id']) + + worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid_v1, FAULT_WORKER_FAIL='permanent') + status, code, _ = wait_db(attached['id'], lambda s, e, a: s == 'failed', 40, 'failed v1') + check(code == 'injected_failure', 'unexpected v1 failure: ' + code) + + compose('stop', 'worker') + reprocessed = json.loads(jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2').stdout) + refused = jobctl('replay', '-job', jid_v1, check=False) + check(refused.returncode != 0, 'older failed version was replayed while v2 exists') + jobs = psql("select pipeline_version || ':' || status from attachment_jobs where attachment_id='%s' order by pipeline_version" % attached['id']) + check(jobs == '1:failed\n2:queued', 'older replay changed job state: ' + jobs.replace('\n', ', ')) + print('replay older failed version while v2 queued: refused, state intact') + + reset_worker() + client.wait_ready(alice, attached, wait_seconds=90) + check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2', + 'v2 did not publish after older replay was refused') + check(psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=2" % attached['id']) == reprocessed['ID'], + 'v2 job identity changed') + print('v2 published after older replay refusal') + + +def main(): + args = arguments('Lab5 level 4 fault injection') + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + + scenarios = [ + scenario_permanent_failure_and_replay, + scenario_replay_older_version_refused, + scenario_replay_reprocessed_version, + scenario_transient_exhaustion, + scenario_crash_after_first_variant, + scenario_crash_after_commit, + scenario_crash_before_side_effect, + scenario_outbox_gap, + scenario_lease_takeover, + ] + + try: + for scenario in scenarios: + scenario(client, alice, chat) + finally: + reset_worker() + reset_relay() + + print('PASS: lab5 level 4 fault injection') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/lab5_variants.py b/student-tests/lab5_variants.py new file mode 100755 index 00000000..d346a99c --- /dev/null +++ b/student-tests/lab5_variants.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Lab5: metadata, bytes and dimensions of all contract variants. + +Run against a live stand: + python3 student-tests/lab5_variants.py --base-url http://localhost:8080 +""" +import hashlib +import struct +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from client import Client, Failure, arguments, check, run # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png' + + +def webp_size(data): + check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'not a WebP container') + + chunk = data[12:16] + if chunk == b'VP8X': + width = 1 + int.from_bytes(data[24:27], 'little') + height = 1 + int.from_bytes(data[27:30], 'little') + return width, height + if chunk == b'VP8 ': + start = data.index(b'\x9d\x01\x2a', 12) + 3 + width = int.from_bytes(data[start:start + 2], 'little') & 0x3fff + height = int.from_bytes(data[start + 2:start + 4], 'little') & 0x3fff + return width, height + if chunk == b'VP8L': + bits = int.from_bytes(data[21:25], 'little') + width = (bits & 0x3fff) + 1 + height = ((bits >> 14) & 0x3fff) + 1 + return width, height + raise Failure('unsupported WebP chunk ' + repr(chunk)) + + +def main(): + args = arguments('Lab5 variant contract') + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + + attached, content = client.attachment(alice, chat) + ready = client.wait_ready(alice, attached) + + by_name = {variant['name']: variant for variant in ready['variants']} + check(set(by_name) == {'thumbnail', 'crop', 'optimized'}, 'unexpected variant set: ' + str(sorted(by_name))) + + expected = { + 'thumbnail': ('image/jpeg', (256, 160)), + 'crop': ('image/jpeg', (128, 128)), + 'optimized': ('image/webp', (320, 200)), + } + + for name, (content_type, dims) in expected.items(): + variant = by_name[name] + response = client.call( + 'GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=' + name, actor=bob, + ) + data = response['bytes'] + check(response['headers'].get_content_type() == content_type, name + ' content type mismatch') + check(variant['content_type'] == content_type, name + ' metadata content type mismatch') + check(len(data) == variant['size_bytes'], name + ' size_bytes mismatch') + check(hashlib.sha256(data).hexdigest() == variant['sha256'], name + ' sha256 mismatch') + + if name == 'optimized': + actual = webp_size(data) + check(actual == dims, name + ' dimensions ' + str(actual)) + check(b'EXIF' not in data[:64], 'optimized WebP must not carry EXIF') + else: + actual = jpeg_size(data) + check(actual == dims, name + ' dimensions ' + str(actual)) + check('Exif' not in data[:64].decode('latin-1', 'ignore'), name + ' must not carry EXIF') + check((variant['width'], variant['height']) == dims, name + ' metadata dimensions mismatch') + + original = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob) + check(original['bytes'] == content, 'original bytes changed') + check(original['headers'].get_content_type() == 'image/png', 'original content type changed') + + print('variants: thumbnail 256x160, crop 128x128, optimized WebP 320x200; metadata == bytes') + print('PASS: lab5 variant contract') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/presign_flow.py b/student-tests/presign_flow.py index 212d97d4..91df5b0d 100755 --- a/student-tests/presign_flow.py +++ b/student-tests/presign_flow.py @@ -52,10 +52,10 @@ def put(url, content, content_type='image/png'): return error.code -def complete(client, actor, attachment_id, digest, expected=200): +def complete(client, actor, attachment_id, digest, expected=202): return client.call( 'POST', '/api/v1/attachments/' + attachment_id + '/complete', expected, - actor=actor, body={'sha256': digest}, schema='Attachment' if expected == 200 else None, + actor=actor, body={'sha256': digest}, schema='Attachment' if expected == 202 else None, ) @@ -73,7 +73,8 @@ def main(): check(put(presigned['upload_url'], content) == 200, 'direct PUT failed') attachment = complete(client, alice, presigned['attachment_id'], digest)['json'] check(attachment['sha256'] == digest and attachment['size_bytes'] == len(content), 'completed metadata mismatch') - print('happy path: direct PUT 200, complete ready, checksum/size match') + attachment = client.wait_ready(alice, attachment) + print('happy path: direct PUT 200, complete queued, ready after worker') client.message(alice, chat, '', attachments=[attachment['id']]) download = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob) @@ -83,6 +84,7 @@ def main(): # Completion is idempotent. again = complete(client, alice, presigned['attachment_id'], digest)['json'] check(again['id'] == attachment['id'], 'repeated complete returned another attachment') + check(again['status'] in ('queued', 'processing', 'ready'), 'repeated complete returned an invalid state') print('repeated complete is idempotent') # A foreign user cannot complete someone else's pending upload. diff --git a/student-tests/tests_helpers.py b/student-tests/tests_helpers.py index dcbb55c9..5821b98c 100644 --- a/student-tests/tests_helpers.py +++ b/student-tests/tests_helpers.py @@ -10,15 +10,32 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -def compose(*args, check=True): +def compose(*args, check=True, env=None): command = [*os.environ.get('COMPOSE_BIN', 'docker compose').split(), '--env-file', '.env', '-f', 'compose.yaml', *args] + environment = {**os.environ, **(env or {})} - return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=check) + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=check, env=environment) + + +def jobctl(*args, check=True): + """Run the operator CLI inside the worker image.""" + return compose('run', '--rm', '--entrypoint', '/usr/local/bin/jobctl', 'worker', *args, check=check) + + +def psql(sql): + """Read a scalar/list from the stand database (operator evidence only).""" + project = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-messenger-lab5') + result = subprocess.run( + ['docker', 'exec', project + '-postgres-1', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql], + cwd=ROOT, capture_output=True, text=True, check=True, + ) + + return result.stdout.strip() def containers_by_hostname(service='api'): """{hostname: {id, name, status}} для контейнеров сервиса.""" - project = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-messenger-lab4') + project = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-messenger-lab5') listed = subprocess.run( [ 'docker', 'ps', '-q', diff --git a/tools/specgen/main.go b/tools/specgen/main.go index cb531e62..67029cf2 100644 --- a/tools/specgen/main.go +++ b/tools/specgen/main.go @@ -1,6 +1,6 @@ // Command specgen produces the OpenAPI document used for code generation. // -// The mandatory contract (contracts/openapi.json, version 1.4.0) describes the +// The mandatory contract (contracts/openapi.json, version 1.5.0) describes the // cross-field rule "text is non-empty OR attachment_ids is non-empty" with an // anyOf. ogen v1.24 cannot generate Go types for two unnamed anyOf branches // with the same name; ignoring the limitation silently drops entire operations. @@ -89,8 +89,8 @@ func derive() ([]byte, error) { } number, _ := lab["number"].(float64) - if number != 4 { - return nil, fmt.Errorf("%s: expected lab number 4, got %v", labPath, lab["number"]) + if number != 5 { + return nil, fmt.Errorf("%s: expected lab number 5, got %v", labPath, lab["number"]) } for _, path := range rewriteTargets {