lab5: очереди и асинхронная обработка изображений

Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
This commit is contained in:
Vladislav Kunetskiy
2026-10-31 14:34:24 +03:00
parent 6c5ba026f1
commit 01b885e846
99 changed files with 10164 additions and 1452 deletions
+54 -1
View File
@@ -97,10 +97,63 @@ S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID=lab4app
S3_SECRET_ACCESS_KEY=lab4app-secret
# Attachment limits from contract 1.4.0.
# Attachment limits from contract 1.5.0.
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Presigned upload lifetime and the sweeper's safety window.
S3_PRESIGN_TTL=5m
S3_SWEEP_INTERVAL=1m
S3_SWEEP_GRACE=10m
# --- Lab5: durable queues and async image processing -------------------------
# Synthetic broker credentials for the local Compose stand. The relay/worker
# connect inside the compose network; the management UI is published only on
# loopback by compose.yaml.
BROKER_USER=messenger
BROKER_PASSWORD=messenger
BROKER_VHOST=messenger
BROKER_URL=amqp://messenger:messenger@broker:5672/messenger
BROKER_EXCHANGE=attachments.jobs
BROKER_QUEUE=attachment.process.v1
BROKER_DLX=attachments.dlx
BROKER_DLQ=attachment.process.dead
BROKER_MANAGEMENT_PORT=15672
BROKER_CPU_LIMIT=0.50
BROKER_MEM_LIMIT=384M
# Worker process bounds. Concurrency=1 is the safe default for a 40 MP image on
# the course VM; raise it together with WORKER_MEM_LIMIT.
WORKER_CONCURRENCY=1
WORKER_PREFETCH=1
WORKER_CPU_LIMIT=1.00
WORKER_MEM_LIMIT=1G
WORKER_SHUTDOWN_TIMEOUT=30s
WORKER_HEALTH_PORT=8081
# Shared job contract. The API chooses the pipeline version for new uploads.
PROCESSING_PIPELINE_VERSION=1
PROCESSING_MAX_ATTEMPTS=5
PROCESSING_JOB_TIMEOUT=120s
PROCESSING_LEASE_TTL=180s
PROCESSING_HEARTBEAT=30s
PROCESSING_RETRY_BASE=2s
PROCESSING_RETRY_CAP=60s
# Outbox relay and lease reaper.
OUTBOX_POLL_INTERVAL=1s
OUTBOX_CLAIM_TTL=30s
OUTBOX_BATCH=20
OUTBOX_REAPER_INTERVAL=15s
OUTBOX_RECONCILE_AFTER=5m
OUTBOX_RETENTION=168h
RELAY_CPU_LIMIT=0.25
RELAY_MEM_LIMIT=128M
RELAY_HEALTH_PORT=8082
# Staging cleanup and admission control.
STAGING_ORPHAN_GRACE=1h
STAGING_FAILED_RETENTION=168h
ADMISSION_MAX_PENDING_PER_USER=20
ADMISSION_MAX_PENDING=200
ADMISSION_MAX_STAGED_BYTES=2147483648
ADMISSION_MAX_CONCURRENT_UPLOADS=8
+8 -2
View File
@@ -47,7 +47,7 @@ jobs:
tags: mtusi-messenger:ci
stand:
name: Lab4 stand scenario
name: Lab5 stand scenario
needs: build
runs-on: ubuntu-latest
timeout-minutes: 20
@@ -68,8 +68,14 @@ jobs:
run: make test
- name: Attachment negative scenarios
run: make test-attachments
- name: Presigned direct upload
- name: Presigned direct upload (async complete)
run: make test-presign
- name: Worker process is separate and healthy
run: docker compose --env-file .env ps worker relay broker
- name: Async worker and broker durability
run: make test-queue
- name: Track 2 duplicate and stale-version harness
run: python3 scripts/lab5_track2.py --reps 1
- name: Parallel idempotency through the LB
run: python3 student-tests/idempotency_parallel.py --base-url http://localhost:8080
- name: Upload, stop the serving replica, download through another
+3
View File
@@ -24,6 +24,9 @@ target/
bin/
/messenger
/s3init
/worker
/relay
/jobctl
obj/
coverage/
coverage.out
+19 -4
View File
@@ -31,17 +31,29 @@ ENV GOMEMLIMIT=1GiB
COPY . .
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath -buildvcs=false -tags timetzdata \
go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /out/messenger ./cmd/messenger \
&& CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath -buildvcs=false -tags timetzdata \
go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /out/migrate ./cmd/migrate \
&& CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath -buildvcs=false -tags timetzdata \
go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /out/s3init ./cmd/s3init
-o /out/s3init ./cmd/s3init \
&& CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /out/worker ./cmd/worker \
&& CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /out/relay ./cmd/relay \
&& CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath -buildvcs=false -tags timetzdata,nodynamic \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /out/jobctl ./cmd/jobctl
FROM alpine:${ALPINE_VERSION} AS runtime
RUN apk add --no-cache ca-certificates wget \
@@ -51,6 +63,9 @@ RUN apk add --no-cache ca-certificates wget \
COPY --from=build --chown=10001:10001 /out/messenger /usr/local/bin/messenger
COPY --from=build --chown=10001:10001 /out/migrate /usr/local/bin/migrate
COPY --from=build --chown=10001:10001 /out/s3init /usr/local/bin/s3init
COPY --from=build --chown=10001:10001 /out/worker /usr/local/bin/worker
COPY --from=build --chown=10001:10001 /out/relay /usr/local/bin/relay
COPY --from=build --chown=10001:10001 /out/jobctl /usr/local/bin/jobctl
COPY --chown=10001:10001 config/dev/config.yaml /etc/messenger/dev/config.yaml
USER 10001:10001
+22
View File
@@ -2,6 +2,7 @@ PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
PIPELINE ?= 2
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
PYTHON_LOCAL = env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 $(PYTHON)
@@ -27,6 +28,15 @@ test-attachments:
test-presign:
$(PYTHON_LOCAL) student-tests/presign_flow.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-queue:
$(PYTHON_LOCAL) student-tests/lab5_async.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-faults:
$(PYTHON_LOCAL) student-tests/lab5_faults.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-track2:
$(PYTHON_LOCAL) scripts/lab5_track2.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON_LOCAL) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -71,6 +81,18 @@ ps:
logs:
$(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) logs -f --tail 100
# Operator views: the failed registry is PostgreSQL, the DLQ is a backstop.
jobs-failed:
$(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) run --rm --entrypoint /usr/local/bin/jobctl worker failed
replay:
@test -n "$(JOB)" || (echo "Set JOB=<job_id>"; exit 2)
$(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) run --rm --entrypoint /usr/local/bin/jobctl worker replay -job "$(JOB)"
reprocess:
@test -n "$(ATTACHMENT)" || (echo "Set ATTACHMENT=<attachment_id>"; exit 2)
$(COMPOSE_BIN) --env-file $(ENV_FILE) -f $(COMPOSE_FILE) run --rm --entrypoint /usr/local/bin/jobctl worker reprocess -attachment "$(ATTACHMENT)" -pipeline "$(PIPELINE)"
.DEFAULT_GOAL := help
GO ?= go
APP_NAME ?= messenger
+198 -17
View File
@@ -1,37 +1,218 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Кунецкий Владислав, БВТ2303
- Целевая оценка; выбранный трек 5: **5, трек 2 (доказанная идемпотентность pipeline)**
- Commit/PR: ветка `lab5`; история коммитов содержит перенос lab1–lab4 и базу контракта 1.5.0
- Использованные источники, библиотеки и помощь ИИ: Go 1.27, PostgreSQL 18, RabbitMQ 4.3.7, SeaweedFS 3.80, HAProxy 3.2, AWS SDK for Go v2, ogen 1.24, sqlc, goose, mockery, `github.com/gen2brain/webp` (libwebp→WASM, CGo-free), `golang.org/x/image`, `github.com/rabbitmq/amqp091-go`. Внешние источники: контракт 1.5.0, `contracts/attachment-states.md`, `contracts/jobs.schema.json`, RabbitMQ quorum queue/delayed retry docs, PostgreSQL `FOR UPDATE SKIP LOCKED`, AWS presigned URL docs. ИИ-ассистент использовался для генерации кода и текста; все сценарии воспроизведены локально, результаты — в `evidence/lab5/`.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker Engine/Desktop с Compose v2, Python 3.10+ для публичных и студенческих тестов,
Go 1.27 — только для unit/integration тестов и `make generate`. Образ собирает приложение сам.
```sh
git clone <repo> mtusi2026lab5 && cd mtusi2026lab5
git switch lab5
cp -n .env.example .env
make check
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# дождаться /health/ready (LB, localhost:8080), затем:
make test
make test-attachments
make test-presign
make test-queue
make test-faults
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
```
Остановка без потери данных: `docker compose --env-file .env down` (named volumes сохраняются).
Намеренное удаление учебных данных: `docker compose --env-file .env down -v`.
Стенд требует ~3–4 ГБ RAM Docker VM для worker (1G) + broker (384M) + существующих сервисов;
в текущем окружении 2 ГБ, поэтому worker работает с `WORKER_CONCURRENCY=1` и `WORKER_MEM_LIMIT=1G`
как верхний предел, а не резервированием. Для серии с 40 Мп изображением нужен запас VM и замер RSS.
`make check` проверяет контракт, Python-синтаксис и S3 identities. `make validate` требует
`requirements-dev.txt`:
```sh
python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt
.venv/bin/python scripts/validate_openapi.py
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```mermaid
flowchart LR
C[client] -->|POST bytes| API[api x N]
API -->|validate + spool| API
API -->|Put staging| S3[(SeaweedFS S3)]
API -->|TX: attachment queued + job + outbox| PG[(PostgreSQL)]
RELAY[relay] -->|claim outbox + publisher confirm| MQ[(RabbitMQ quorum + DLQ)]
MQ -->|at-least-once delivery| W[worker]
W -->|read source, verify sha| S3
W -->|Put original + variants| S3
W -->|TX: variants + ready/retry/failed| PG
W -->|ack after commit| MQ
API -->|GET status/variants| PG
API -->|GET content| S3
RELAY -->|lease reaper| PG
```
- **API** принимает байты, валидирует PNG/JPEG по содержимому (bytes, пиксели, MIME, filename),
пишет оригинал в приватный S3 staging, затем одной транзакцией создаёт `attachments(status=queued)`,
`attachment_jobs` и `outbox_messages`. Только после этого возвращается **202** со снимком `queued`.
API не публикует в broker и не делает resize.
- **Relay** — отдельный процесс: читает outbox через `FOR UPDATE SKIP LOCKED`, публикует persistent
сообщение с publisher confirm, помечает `published_at` только после ack. Там же lease reaper:
просроченные `processing` возвращаются в `queued` с новым attempt, а исчерпавшие бюджет — в `failed`.
Дополнительно reaper переиздаёт `queued` jobs без unpublished outbox-строки (потерянная доставка,
dead-letter или data loss broker-а); worker CAS делает дубль безопасным.
- **Worker** — отдельный процесс/контейнер. Claim через CAS, heartbeat, обработка, запись S3,
затем одна транзакция результата; ack только после commit. Падение до ack приводит к redelivery;
повтор видит готовый эффект и делает no-op ack.
- **PostgreSQL** — истина о состоянии/намерении. **S3** — байты. **RabbitMQ** — durable transport
at-least-once, не источник истины. Границы: 202 = staging + TX intent; publisher confirm ≠ consumer ack.
- Staging: `staging/uploads/<chat>/<attachment>`; presign проверенный объект копируется в
`staging/verified/...`; финальный оригинал — `attachments/<chat>/<attachment>/original`;
варианты — `attachments/<chat>/<attachment>/variants/v<pipeline>/<name>.<ext>`.
- Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`: PK в `attachment_variants`,
deterministic S3 keys, CAS по lease/attempt. `ready` терминален; старая версия не перезаписывает новую.
- `attachments.pipeline_version` — **опубликованная** версия. `jobctl reprocess` создаёт job новой
версии, но переключает `pipeline_version` и variants только в транзакции успешного завершения;
до этого API продолжает отдавать старую готовую версию.
- Presign — факультативное расширение вне OAS: presign создаёт internal `pending`-строку; complete
проверяет байты, копирует их в server-owned verified staging и возвращает 202 queued.
- Sweeper status-aware: staging queued/processing не удаляется; ready staging удаляется после
безопасной фиксации; failed — после retention; orphan staging — после orphan grace. Referenced set
включает финальные объекты, variants и активный staging; keyset-пагинация, fail-closed.
- Admission control ограничивает число незавершённых rows (включая `pending` presign) на
пользователя/стенд и объём staged bytes (429 `rate_limited`). Проверка повторяется внутри
транзакции под PostgreSQL advisory lock, поэтому параллельные запросы не обходят cap; presign
использует тот же механизм. Число одновременных upload-spool на процесс ограничено
`ADMISSION_MAX_CONCURRENT_UPLOADS`, что ограничивает и временный диск API.
### Соответствие `.env.example` реальной конфигурации
`BROKER_URL`, `WORKER_CONCURRENCY`, `WORKER_PREFETCH`, `PROCESSING_*`, `OUTBOX_*`, `STAGING_*`,
`ADMISSION_*` читаются viper-ом (`section.key` → `SECTION_KEY`). `S3_*`, `MAX_UPLOAD_BYTES`,
`MAX_IMAGE_PIXELS`, `POSTGRES_*`, `REDIS_*` сохранены с lab4. Compose подменяет `localhost`
на имена сервисов для контейнеров; значения `.env` остаются локальными для запуска с хоста.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke, upload 201→202, thumbnail | `make test` | PASS lab5: 202→queued→ready, 256×160 JPEG, sha/size | `evidence/lab5/smoke.txt` |
| 202 после durable staging, worker отдельно | `make test-queue` (`lab5_async.py`) | worker stopped: 202 queued, content 409, ссылка на сообщение; после старта ready | `evidence/lab5/async.txt` |
| Broker restart с volume | `lab5_async.py` | upload при остановленном broker → 202; после старта broker+relay ready | `evidence/lab5/async.txt` |
| Полный restart стека | `lab5_async.py` | ready-вложение и original сохранились | `evidence/lab5/async.txt` |
| Варианты: thumbnail/crop/optimized | `make test-queue`/`lab5_variants.py` | thumbnail 256×160 JPEG, crop 128×128 JPEG, optimized WebP 320×200; metadata == bytes; EXIF отсутствует | `evidence/lab5/variants.txt` |
| Состояния queued/processing/ready/failed | `lab5_async.py`, `lab5_faults.py` | queued/ready и failed/error_code наблюдаются через API | `evidence/lab5/async.txt`, `faults.txt` |
| 415 на невалидное изображение | `make test-attachments` | пустое тело, IDAT-CRC, MIME spoof, не-zlib IDAT, пустой растр, недопустимый scanline-фильтр, несовместимые color type/bit depth, дубликаты IHDR, мусор после zlib, битый tRNS, PLTE сверх bit depth, tRNS сверх PLTE — 415 до 202 | `evidence/lab5/attachments-negative.txt` |
| Original неизменен | smoke + variants | скачанные байты и sha совпадают с загруженными | `evidence/lab5/smoke.txt`, `variants.txt` |
| Retry/backoff/classification | `make test-faults` | permanent → failed сразу; transient → 5 attempts, 4 retry events, `retry_exhausted`; DLQ | `evidence/lab5/faults.txt` |
| DLQ/registry + replay | `lab5_faults.py` | failed виден в `jobctl failed`; replay → ready; replay ready refused | `evidence/lab5/faults.txt` |
| Crash после записи объекта до ack | `lab5_faults.py` | redelivery, attempts=2, тот же набор из трёх variants | `evidence/lab5/faults.txt` |
| Crash после claim до side effect | `lab5_faults.py` | lease/redelivery, attempts=2, ready | `evidence/lab5/faults.txt` |
| Crash после commit до ack | `lab5_faults.py` | ready сохранился, повтор — no-op, attempt не вырос | `evidence/lab5/faults.txt` |
| Outbox gap COMMIT→publish | `lab5_faults.py` | unpublished row → relay crash → recovery → published/ready | `evidence/lab5/faults.txt` |
| Lease зависшего processing | `lab5_faults.py` | pause worker → lease expiry → requeue → fence stale → ready | `evidence/lab5/faults.txt` |
| Persistence lab2+ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS с broker/worker/relay в hook | `evidence/lab5/persistence.txt` |
| Failover lab3+ | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 99 успешных записей, 0 ошибок, recovery 0.22 s | `evidence/lab5/failover.txt` |
| Idempotency lab3+ | `make test-idempotency` | 1 accepted, 11 conflicts, stable replay | `evidence/lab5/idempotency.txt` |
| LB исключает not-ready реплику | `make test-lb-readiness` | candidate 200→503, LB DOWN в пределах 30 s, 40 запросов без candidate | `evidence/lab5/lb-readiness-detail.txt` |
| Трек 2: duplicate/stale/version | `python3 scripts/lab5_track2.py --reps 3` | 6/6 PASS: duplicate before effect, v1→v2, stale v1 no-op | `evidence/lab5/track2/` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Параметры кодирования вариантов
| pipeline_version | thumbnail/crop JPEG | optimized WebP | WebP method | EXIF/alpha |
| --- | --- | --- | --- | --- |
| 1 | quality 85 | quality 80 | 2 | EXIF orientation применяется; metadata удаляется повторным encode; alpha компонуется на белый для JPEG |
| 2 | quality 70 | quality 65 | 4 | то же |
Параметры закомпилированы в `internal/imageproc.Pipeline` и неизменны для версии; retry не может
получить другие байты. Optimized не увеличивает изображение, fit ≤1280×1280.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидали | Что наблюдали |
| --- | --- | --- |
| Worker остановлен до upload | 202 queued, content 409, работа не теряется | 202, статус queued >2 s, после старта ready |
| Broker остановлен до upload | 202, outbox intent сохраняется | 202; после старта broker+relay ready |
| Relay убит после COMMIT до publish | outbox остаётся, повтор публикует | unpublished→published, ready, дублей эффекта нет |
| Worker убит после первого variant | redelivery дописывает тот же набор | attempts=2, три variants, original не изменён |
| Worker убит после commit до ack | ready, повтор — no-op | ready, attempt=1 |
| Worker paused (lease) | reaper вернёт работу, stale worker не запишет | requeue, новый attempt, ready |
| Permanent error | failed сразу, error_code, DLQ | failed/injected_failure, attempts=1 |
| Transient errors | 5 attempts, backoff+jitter, failed | attempts=5, `retry_exhausted`, DLQ |
| Failed replay | failed→queued→ready без дубля | ready, один thumbnail |
| v1 ready → reprocess v2 → old v1 delivery | v2 не перезаписывается | pv=2, variants v2, stale v1 ack |
| Reprocess queued при остановленном worker | v2 берёт проверенный staging, а не несуществующий original | pv=2 ready, v1 superseded, событие superseded записано |
| Reprocess v2 падает до публикации | вложение терминально failed, не вечный queued | failed/error_code, active jobs 0 |
| Replay упавшей v2 до публикации | API не показывает stale failed, работа возвращается | job queued, attachment queued, затем ready:v2 |
| Replay старой failed-v1 при наличии v2 | команда отклонена, источник v2 не потерян | v1 failed, v2 queued → ready:v2 |
| Version-stale queued job после публикации v2 | задание терминально superseded, не переиздаётся | claim → superseded, reconcile не создаёт работу |
| Reprocess на неподдерживаемую версию | команда отклонена, v1-задание не тронуто | jobctl exit != 0, v1 queued → ready |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Измерения: recovery времени и counts — из `evidence/lab5/faults.txt`, `track2/runs.csv`.
Стенд: 2 vCPU, 2 ГБ Docker VM; синтетические пользователи/чат; fixture 320×200 PNG.
Выводы — для локального Compose, не production SLA.
## Ограничения и следующие шаги
## Ограничения
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Доставка **at-least-once**, не exactly-once: crash между publish и confirm/ack даёт дубликат;
эффект идемпотентен, но глобального exactly-once нет.
- Single-node RabbitMQ: durability при рестарте с volume, но не HA/replication.
- 40 Мп и 16-bit PNG не прогонялись в этой серии; worker имеет pixel cap, таймаут, watchdog
и memory limit, но пиковый RSS на 40 Мп нужно измерять отдельно.
- Track 2 — correctness harness (duplicate/stale/version + fault suite), не нагрузочный эксперимент;
interleavings sampled, не исчерпаны.
- Presign — optional extension; его ответ 202 и проверенный staging описаны в отчёте, но публичный
контракт его не требует.
- Rate limiting — admission cap с транзакционным reservation, не промышленный token bucket.
- Presign подписывает URL статическим локальным app-identity; S3 API опубликован только на
loopback для учебного стенда. В production нужен отдельный least-privilege signer и запрет
локального доступа к bucket.
- API readiness проверяет PostgreSQL, не S3/broker: при отказе S3 upload отвечает 503, но
маршрутизация LB и чтение метаданных продолжают работать.
- Reconciliation потерянной доставки запускается через `OUTBOX_RECONCILE_AFTER` (5m), поэтому
dead-letter не теряет intent, но восстановление не мгновенное.
- Failpoints включаются только `FAULTS_ENABLED=1` при `APP_ENV=dev`; в обычном стенде инертны.
- `x-delivery-limit` — immutable queue argument. На чистом стенде создаётся 100; при обновлении
существующего volume очередь нужно пересоздать (или применить policy) до старта worker/relay.
- Старые lab4 ready-вложения не переобрабатываются миграцией; для них нужен ручной
`jobctl reprocess` (backfill в эту серию не входил).
## Трек 2: гипотеза, методика, результаты
**Гипотеза.** При at-least-once доставке, effect key `(attachment_id, pipeline_version, variant)`,
deterministic S3 keys, serving-версии, меняющейся только после commit всех v2 variants, и ack
stale-доставок pipeline не теряет принятую работу и не перезаписывает новый результат старым.
**Методика.** `scripts/lab5_track2.py --reps 3`: для каждой репетиции создаются новые пользователи/чат;
(1) worker остановлен, upload, две duplicate-публикации одного job, затем worker стартует;
(2) v1 ready → `jobctl reprocess -pipeline 2` → v2 ready → публикуется старое v1-сообщение.
Проверяются: уникальность variant rows, отсутствие регрессии ready, монотонность `pipeline_version`,
неизменность original, no-op stale job. Fault suite дополнительно проверяет crash окна и lease.
**Результаты.** 6/6 сценариев PASS (`evidence/lab5/track2/runs.csv`), все fault-сценарии PASS.
Исходные данные — `evidence/lab5/track2/run.txt`, `summary.md`, `runs.csv`.
**Почему не exactly-once.** Publisher confirm подтверждает приём брокером, consumer ack — устойчивый
эффект; crash между ними создаёт повторную доставку. Идемпотентный effect key и CAS делают повтор
безопасным, но не устраняют дубликаты доставки.
## Ответы на вопросы защиты
- **Publisher confirm vs consumer ack:** confirm — RabbitMQ durable accepted; ack — worker записал
эффект. 202 не ждёт ни того, ни другого.
- **Что сохранено к 202:** полный проверенный original в private S3 staging, attachment metadata
`queued`, job intent и outbox row в одной TX. Thumbnail/final original/broker publish могут ещё
не существовать.
- **Почему redelivery нормальна:** at-least-once; effect key, deterministic keys, CAS lease/attempt,
`ready`-no-op. Старый attempt/версия только ack-ается.
- **Stuck vs slow:** heartbeat продлевает lease; живой медленный worker не забирается. Мёртвый
перестаёт продлевать; reaper возвращает работу, старый CAS-fenced.
- **Replay:** только failed, явный CLI, audit events, budget attempt-ов; ready replay запрещён.
+215
View File
@@ -0,0 +1,215 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log/slog"
"os"
"time"
"github.com/google/uuid"
"mtusi-messenger/internal/config"
"mtusi-messenger/internal/imageproc"
"mtusi-messenger/internal/logger"
"mtusi-messenger/internal/storage/postgres"
)
func main() {
if len(os.Args) < 2 {
usage()
os.Exit(2)
}
cfg, err := config.Load()
if err != nil {
fmt.Fprintln(os.Stderr, "load config:", err)
os.Exit(1)
}
log := logger.New(logger.WithLevel(cfg.SlogLevel()), logger.WithFormat(logger.FormatConsole))
slog.SetDefault(log)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second)
if err != nil {
fmt.Fprintln(os.Stderr, "connect database:", err)
cancel()
os.Exit(1) //nolint:gocritic // main exits after DB setup failure
}
defer store.Close()
repo := store.Pipeline()
switch os.Args[1] {
case "failed":
runFailed(ctx, repo)
case "replay":
runReplay(ctx, repo, cfg)
case "reprocess":
runReprocess(ctx, repo, cfg)
case "duplicate":
runDuplicate(ctx, repo)
case "inspect":
runInspect(ctx, repo)
default:
usage()
os.Exit(2)
}
}
func usage() {
fmt.Fprintln(os.Stderr, `usage: jobctl <command> [flags]
failed -limit N
replay -job JOB_ID [-attempts N]
reprocess -attachment ATTACHMENT_ID -pipeline N
duplicate -attachment ATTACHMENT_ID | -job JOB_ID
inspect -attachment ATTACHMENT_ID`)
}
func runFailed(ctx context.Context, repo *postgres.PipelineRepository) {
flags := flag.NewFlagSet("failed", flag.ExitOnError)
limit := flags.Int("limit", 50, "maximum jobs")
_ = flags.Parse(os.Args[2:])
jobs, err := repo.ListFailedJobs(ctx, *limit)
if err != nil {
fail(err)
}
printJSON(jobs)
}
func runReplay(ctx context.Context, repo *postgres.PipelineRepository, cfg config.Config) {
flags := flag.NewFlagSet("replay", flag.ExitOnError)
jobID := flags.String("job", "", "job id")
attempts := flags.Int("attempts", cfg.Processing.MaxAttempts, "additional attempts")
_ = flags.Parse(os.Args[2:])
id, err := uuid.Parse(*jobID)
if err != nil {
fail(fmt.Errorf("invalid job id: %w", err))
}
job, err := repo.ReplayJob(ctx, id, *attempts)
if err != nil {
fail(err)
}
printJSON(job)
}
func runReprocess(ctx context.Context, repo *postgres.PipelineRepository, cfg config.Config) {
flags := flag.NewFlagSet("reprocess", flag.ExitOnError)
attachmentID := flags.String("attachment", "", "attachment id")
version := flags.Int("pipeline", 2, "target pipeline version")
_ = flags.Parse(os.Args[2:])
if _, ok := imageproc.Pipeline(*version); !ok {
fail(fmt.Errorf("unsupported pipeline version %d", *version))
}
id, err := uuid.Parse(*attachmentID)
if err != nil {
fail(fmt.Errorf("invalid attachment id: %w", err))
}
jobID, err := uuid.NewV7()
if err != nil {
fail(err)
}
job, err := repo.Reprocess(ctx, id, *version, cfg.Processing.MaxAttempts, jobID)
if err != nil {
fail(err)
}
printJSON(job)
}
func runDuplicate(ctx context.Context, repo *postgres.PipelineRepository) {
flags := flag.NewFlagSet("duplicate", flag.ExitOnError)
attachmentID := flags.String("attachment", "", "attachment id")
jobID := flags.String("job", "", "specific job id (old pipeline version)")
_ = flags.Parse(os.Args[2:])
if *jobID != "" {
id, err := uuid.Parse(*jobID)
if err != nil {
fail(fmt.Errorf("invalid job id: %w", err))
}
message, err := repo.DuplicateJob(ctx, id)
if err != nil {
fail(err)
}
printJSON(message)
return
}
id, err := uuid.Parse(*attachmentID)
if err != nil {
fail(fmt.Errorf("attachment or job id is required: %w", err))
}
message, err := repo.Duplicate(ctx, id)
if err != nil {
fail(err)
}
printJSON(message)
}
func runInspect(ctx context.Context, repo *postgres.PipelineRepository) {
flags := flag.NewFlagSet("inspect", flag.ExitOnError)
attachmentID := flags.String("attachment", "", "attachment id")
_ = flags.Parse(os.Args[2:])
id, err := uuid.Parse(*attachmentID)
if err != nil {
fail(fmt.Errorf("invalid attachment id: %w", err))
}
attachment, err := repo.GetForID(ctx, id)
if err != nil {
fail(err)
}
variants, err := repo.ListVariants(ctx, id, attachment.PipelineVersion)
if err != nil {
fail(err)
}
var jobOutput any
job, jobErr := repo.GetJobByAttachment(ctx, id, attachment.PipelineVersion)
if jobErr == nil {
jobOutput = job
}
printJSON(map[string]any{
"attachment": attachment,
"variants": variants,
"job": jobOutput,
})
}
func printJSON(value any) {
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if err := encoder.Encode(value); err != nil {
fail(err)
}
}
func fail(err error) {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
+15 -7
View File
@@ -74,23 +74,31 @@ func run(cfg config.Config, log *slog.Logger) error {
return fmt.Errorf("connect object store: %w", err)
}
attachmentRepository := store.Attachments()
pipelineRepository := store.Pipeline()
attachmentService := attachment.NewService(
attachmentRepository,
pipelineRepository,
objectStore,
attachment.Config{
MaxUploadBytes: cfg.S3.MaxUploadBytes,
MaxImagePixels: cfg.S3.MaxImagePixels,
PresignTTL: cfg.S3.PresignTTL,
MaxUploadBytes: cfg.S3.MaxUploadBytes,
MaxImagePixels: cfg.S3.MaxImagePixels,
PresignTTL: cfg.S3.PresignTTL,
PipelineVersion: cfg.Processing.PipelineVersion,
MaxAttempts: cfg.Processing.MaxAttempts,
MaxPendingPerUser: cfg.Admission.MaxPendingPerUser,
MaxPending: cfg.Admission.MaxPending,
MaxStagedBytes: cfg.Admission.MaxStagedBytes,
MaxConcurrent: cfg.Admission.MaxConcurrent,
},
log,
)
sweeper := attachment.NewSweeper(
attachmentRepository,
pipelineRepository,
objectStore,
store,
cfg.S3.SweepGrace,
cfg.Staging.OrphanGrace,
cfg.Staging.FailedRetention,
cfg.S3.SweepInterval,
log,
)
@@ -151,7 +159,7 @@ func run(cfg config.Config, log *slog.Logger) error {
rest.NewHandler(
user.NewService(store.Users()),
chat.NewService(chatRepo),
message.NewService(store.Messages(), attachmentRepository),
message.NewService(store.Messages(), pipelineRepository),
attachmentService,
health.New(pingers...),
sessionService,
+262
View File
@@ -0,0 +1,262 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/config"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/logger"
"mtusi-messenger/internal/outbox"
"mtusi-messenger/internal/storage/postgres"
)
const maxUnpublishedAge = 10 * time.Minute
func main() {
log := logger.New()
cfg, err := config.Load()
if err != nil {
log.Error("load config", slog.Any("error", err))
os.Exit(1)
}
log = logger.New(
logger.WithLevel(cfg.SlogLevel()),
logger.WithFormat(cfg.LoggerFormat()),
)
slog.SetDefault(log)
if err := run(cfg, log); err != nil {
log.Error("shutdown with error", slog.Any("error", err))
os.Exit(1)
}
}
type publisher struct {
cfg broker.Config
log *slog.Logger
mu sync.Mutex
conn *broker.Client
}
func (p *publisher) client() (*broker.Client, error) {
p.mu.Lock()
defer p.mu.Unlock()
if p.conn != nil && p.conn.Connected() {
return p.conn, nil
}
if p.conn != nil {
_ = p.conn.Close()
p.conn = nil
}
conn, err := broker.Dial(p.cfg)
if err != nil {
return nil, err
}
p.conn = conn
return conn, nil
}
func (p *publisher) PublishConfirmed(
ctx context.Context,
routingKey string,
message domain.JobMessage,
) error {
conn, err := p.client()
if err != nil {
return err
}
if err := conn.PublishConfirmed(ctx, routingKey, message); err != nil {
p.mu.Lock()
if p.conn == conn {
_ = p.conn.Close()
p.conn = nil
}
p.mu.Unlock()
return err
}
return nil
}
func (p *publisher) Connect() error {
_, err := p.client()
return err
}
func (p *publisher) Connected() bool {
p.mu.Lock()
defer p.mu.Unlock()
return p.conn != nil && p.conn.Connected()
}
func (p *publisher) Close() {
p.mu.Lock()
defer p.mu.Unlock()
if p.conn != nil {
_ = p.conn.Close()
p.conn = nil
}
}
func brokerConfig(cfg config.Config) broker.Config {
return broker.Config{
URL: cfg.Broker.URL,
Exchange: cfg.Broker.Exchange,
DLX: cfg.Broker.DLX,
Queue: cfg.Broker.Queue,
DLQ: cfg.Broker.DLQ,
ConnectTimeout: cfg.Broker.ConnectTimeout,
ConfirmTimeout: cfg.Broker.ConfirmTimeout,
Heartbeat: cfg.Broker.Heartbeat,
Prefetch: cfg.Broker.Prefetch,
}
}
func run(cfg config.Config, log *slog.Logger) error {
if cfg.Broker.URL == "" {
return errors.New("BROKER_URL is required for relay")
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second)
if err != nil {
return fmt.Errorf("connect database: %w", err)
}
defer store.Close()
pub := &publisher{cfg: brokerConfig(cfg), log: log}
defer pub.Close()
if err := pub.Connect(); err != nil {
log.Warn("broker is not reachable yet; relay will retry", slog.Any("error", err))
}
relay := outbox.New(store.Pipeline(), pub, outbox.Config{
Owner: "relay-" + cfg.Broker.Queue,
PollInterval: cfg.Outbox.PollInterval,
ClaimTTL: cfg.Outbox.ClaimTTL,
Batch: cfg.Outbox.Batch,
ReaperInterval: cfg.Outbox.ReaperInterval,
ReconcileAfter: cfg.Outbox.ReconcileAfter,
Retention: cfg.Outbox.Retention,
}, log)
relayCtx, cancelRelay := context.WithCancel(ctx)
defer cancelRelay()
relayDone := make(chan struct{})
go func() {
defer close(relayDone)
relay.Run(relayCtx)
}()
server := healthServer(cfg.Relay.HealthAddr, store, pub, relay, log)
go func() {
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("health server failed", slog.Any("error", err))
}
}()
<-ctx.Done()
cancelRelay()
<-relayDone
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return server.Shutdown(shutdownCtx)
}
func healthServer(
addr string,
store *postgres.Store,
pub *publisher,
relay *outbox.Relay,
_ *slog.Logger,
) *http.Server {
mux := http.NewServeMux()
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) {
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
})
mux.HandleFunc("/health/ready", func(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
if err := store.Ping(ctx); err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
if !pub.Connected() {
if err := pub.Connect(); err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
}
oldest, err := relay.Health(ctx)
if err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
if oldest > maxUnpublishedAge {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{
"status": "unavailable",
"outbox_oldest_seconds": int(oldest.Seconds()),
})
return
}
writeHealth(w, http.StatusOK, map[string]any{
"status": "ok",
"outbox_oldest_seconds": int(oldest.Seconds()),
})
})
return &http.Server{
Addr: addr,
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
}
}
func writeHealth(w http.ResponseWriter, status int, body map[string]any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(body)
}
+231
View File
@@ -0,0 +1,231 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/config"
"mtusi-messenger/internal/logger"
"mtusi-messenger/internal/platform/objectstore"
"mtusi-messenger/internal/storage/postgres"
"mtusi-messenger/internal/worker"
)
func main() {
log := logger.New()
cfg, err := config.Load()
if err != nil {
log.Error("load config", slog.Any("error", err))
os.Exit(1)
}
log = logger.New(
logger.WithLevel(cfg.SlogLevel()),
logger.WithFormat(cfg.LoggerFormat()),
)
slog.SetDefault(log)
if err := run(cfg, log); err != nil {
log.Error("shutdown with error", slog.Any("error", err))
os.Exit(1)
}
}
func run(cfg config.Config, log *slog.Logger) error {
if cfg.Broker.URL == "" {
return errors.New("BROKER_URL is required for worker")
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second)
if err != nil {
return fmt.Errorf("connect database: %w", err)
}
defer store.Close()
objectStore, err := objectstore.New(objectstore.Config{
Endpoint: cfg.S3.Endpoint,
PublicEndpoint: cfg.S3.PublicEndpoint,
Region: cfg.S3.Region,
Bucket: cfg.S3.Bucket,
AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey,
UsePathStyle: cfg.S3.UsePathStyle,
Timeout: cfg.S3.Timeout,
})
if err != nil {
return fmt.Errorf("connect object store: %w", err)
}
workerInstance := worker.New(store.Pipeline(), objectStore, worker.Config{
Concurrency: cfg.Worker.Concurrency,
LeaseTTL: cfg.Processing.LeaseTTL,
Heartbeat: cfg.Processing.Heartbeat,
JobTimeout: cfg.Processing.JobTimeout,
RetryBase: cfg.Processing.RetryBase,
RetryCap: cfg.Processing.RetryCap,
MaxUploadBytes: cfg.S3.MaxUploadBytes,
MaxImagePixels: cfg.S3.MaxImagePixels,
}, log)
server := healthServer(cfg.Worker.HealthAddr, store, objectStore, workerInstance)
var serverWait sync.WaitGroup
serverWait.Add(1)
go func() {
defer serverWait.Done()
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("health server failed", slog.Any("error", err))
}
}()
runCtx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
consumerDone := make(chan struct{})
go func() {
defer close(consumerDone)
consumeWithReconnect(runCtx, cfg, workerInstance, log)
}()
<-ctx.Done()
cancelRun()
<-consumerDone
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Worker.ShutdownTimeout)
defer cancel()
_ = server.Shutdown(shutdownCtx)
serverWait.Wait()
return nil
}
func consumeWithReconnect(
ctx context.Context,
cfg config.Config,
workerInstance *worker.Worker,
log *slog.Logger,
) {
for ctx.Err() == nil {
client, err := broker.Dial(brokerConfig(cfg))
if err != nil {
log.WarnContext(ctx, "broker dial failed, retrying", slog.Any("error", err))
if !sleepCtx(ctx, 2*time.Second) {
return
}
continue
}
err = workerInstance.Run(ctx, client)
_ = client.Close()
if ctx.Err() != nil {
return
}
if err != nil {
log.WarnContext(ctx, "consumer stopped, reconnecting", slog.Any("error", err))
}
if !sleepCtx(ctx, 2*time.Second) {
return
}
}
}
func sleepCtx(ctx context.Context, d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
func brokerConfig(cfg config.Config) broker.Config {
return broker.Config{
URL: cfg.Broker.URL,
Exchange: cfg.Broker.Exchange,
DLX: cfg.Broker.DLX,
Queue: cfg.Broker.Queue,
DLQ: cfg.Broker.DLQ,
ConnectTimeout: cfg.Broker.ConnectTimeout,
ConfirmTimeout: cfg.Broker.ConfirmTimeout,
Heartbeat: cfg.Broker.Heartbeat,
Prefetch: cfg.Worker.Prefetch,
}
}
func healthServer(
addr string,
store *postgres.Store,
objectStore objectstore.Store,
workerInstance *worker.Worker,
) *http.Server {
mux := http.NewServeMux()
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) {
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
})
mux.HandleFunc("/health/ready", func(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()
if err := store.Ping(ctx); err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
if err := objectStore.Ping(ctx); err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
running, connected := workerInstance.Ready()
if !running || !connected {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
})
return &http.Server{
Addr: addr,
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
}
}
func writeHealth(w http.ResponseWriter, status int, body map[string]any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(body)
}
+172 -5
View File
@@ -1,4 +1,5 @@
# Lab4 stand: two API replicas behind HAProxy, PostgreSQL, Redis and SeaweedFS S3.
# Lab5 stand: API replicas behind HAProxy, PostgreSQL, Redis, SeaweedFS S3,
# a durable RabbitMQ broker, one outbox relay and a separate image worker.
#
# cp -n .env.example .env
# docker compose --env-file .env up -d --build
@@ -8,13 +9,13 @@
# reachable from the host via `docker compose exec` / `docker compose logs`.
# No container_name anywhere: replica hostnames stay the short container IDs,
# which the application reports back as X-Instance-Id.
name: mtusi-messenger-lab4
name: mtusi-messenger-lab5
# The image is built once by the api service. migrate and s3-init only consume
# it; duplicating `build` here makes Compose compile the same Go tree in parallel
# and OOMs Docker Desktop's default VM.
x-app: &app
image: mtusi-messenger:lab4
image: mtusi-messenger:lab5
services:
postgres:
@@ -42,7 +43,7 @@ services:
restart: unless-stopped
redis:
image: redis:7.4-alpine
image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
command:
- redis-server
- --appendonly
@@ -100,6 +101,35 @@ services:
memory: 640M
restart: unless-stopped
# Durable job transport. Quorum queues, publisher confirms and a DLQ live in
# the broker; PostgreSQL's outbox remains the source of truth for intent.
broker:
image: rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528
hostname: broker
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-messenger}
RABBITMQ_DEFAULT_VHOST: ${BROKER_VHOST:-messenger}
volumes:
- rabbitmq-data:/var/lib/rabbitmq
ports:
# Management UI only on loopback, for operator evidence. AMQP stays internal.
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 12
start_period: 30s
deploy:
resources:
limits:
cpus: "${BROKER_CPU_LIMIT:-0.50}"
memory: ${BROKER_MEM_LIMIT:-384M}
stop_grace_period: 30s
restart: unless-stopped
# One-shot, idempotent bucket bootstrap and identity probe.
s3-init:
<<: *app
@@ -184,6 +214,19 @@ services:
S3_SWEEP_GRACE: ${S3_SWEEP_GRACE:-10m}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1}
PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5}
PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s}
PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s}
PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s}
PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s}
PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s}
STAGING_ORPHAN_GRACE: ${STAGING_ORPHAN_GRACE:-1h}
STAGING_FAILED_RETENTION: ${STAGING_FAILED_RETENTION:-168h}
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
# Размер пула соединений ОДНОЙ реплики; общий расход = реплики × значение.
# Это один из общих ресурсов, который масштабирование расходует первым.
POSTGRES_MAX_CONNS: "${POSTGRES_MAX_CONNS:-10}"
@@ -217,10 +260,133 @@ services:
start_period: 5s
restart: unless-stopped
# Separate worker process. It is intentionally not published and can be
# stopped/scaled independently from the API; stopping it must not stop 202.
worker:
<<: *app
entrypoint: ["/usr/local/bin/worker"]
env_file:
- path: .env
required: false
environment:
APP_CONFIG: /etc/messenger/dev/config.yaml
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
POSTGRES_SSL_MODE: disable
S3_ENDPOINT: http://seaweedfs:8333
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret}
S3_USE_PATH_STYLE: "true"
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
WORKER_SHUTDOWN_TIMEOUT: ${WORKER_SHUTDOWN_TIMEOUT:-30s}
WORKER_HEALTH_ADDR: ":8081"
FAULTS_ENABLED: ${FAULTS_ENABLED:-0}
FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-}
FAULT_WORKER_AFTER_CLAIM: ${FAULT_WORKER_AFTER_CLAIM:-}
FAULT_WORKER_AFTER_FIRST_VARIANT: ${FAULT_WORKER_AFTER_FIRST_VARIANT:-}
FAULT_WORKER_AFTER_PUT: ${FAULT_WORKER_AFTER_PUT:-}
FAULT_WORKER_AFTER_COMMIT: ${FAULT_WORKER_AFTER_COMMIT:-}
FAULT_WORKER_FAIL: ${FAULT_WORKER_FAIL:-}
FAULT_WORKER_SLEEP_AFTER_CLAIM: ${FAULT_WORKER_SLEEP_AFTER_CLAIM:-}
PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1}
PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5}
PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s}
PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s}
PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s}
PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s}
PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s}
ports:
- "127.0.0.1:${WORKER_HEALTH_PORT:-8081}:8081"
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
broker:
condition: service_healthy
deploy:
resources:
limits:
cpus: "${WORKER_CPU_LIMIT:-1.00}"
memory: ${WORKER_MEM_LIMIT:-1G}
stop_grace_period: 40s
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8081/health/ready"]
interval: 10s
timeout: 5s
retries: 12
start_period: 10s
restart: unless-stopped
# Outbox publisher + lease reaper. One replica: duplicate relays are safe but
# a single owner keeps the evidence and the claim reaper simple.
relay:
<<: *app
entrypoint: ["/usr/local/bin/relay"]
env_file:
- path: .env
required: false
environment:
APP_CONFIG: /etc/messenger/dev/config.yaml
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
POSTGRES_SSL_MODE: disable
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger}
BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1}
BROKER_DLQ: ${BROKER_DLQ:-attachment.process.dead}
OUTBOX_POLL_INTERVAL: ${OUTBOX_POLL_INTERVAL:-1s}
OUTBOX_CLAIM_TTL: ${OUTBOX_CLAIM_TTL:-30s}
OUTBOX_BATCH: ${OUTBOX_BATCH:-20}
OUTBOX_REAPER_INTERVAL: ${OUTBOX_REAPER_INTERVAL:-15s}
OUTBOX_RECONCILE_AFTER: ${OUTBOX_RECONCILE_AFTER:-5m}
OUTBOX_RETENTION: ${OUTBOX_RETENTION:-168h}
RELAY_HEALTH_ADDR: ":8082"
FAULTS_ENABLED: ${FAULTS_ENABLED:-0}
FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-}
FAULT_OUTBOX_AFTER_COMMIT: ${FAULT_OUTBOX_AFTER_COMMIT:-}
ports:
- "127.0.0.1:${RELAY_HEALTH_PORT:-8082}:8082"
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
deploy:
replicas: 1
resources:
limits:
cpus: "${RELAY_CPU_LIMIT:-0.25}"
memory: ${RELAY_MEM_LIMIT:-128M}
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/health/ready"]
interval: 10s
timeout: 5s
retries: 12
start_period: 10s
restart: unless-stopped
# Public entry point. Active /health/ready probes remove a not-ready replica from
# rotation entirely; infra/lb/haproxy.cfg holds the upstream, checks and retries.
lb:
image: haproxy:3.2-alpine
image: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
volumes:
- ./infra/lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
@@ -241,3 +407,4 @@ volumes:
postgres18-data:
redis7-data:
seaweedfs-data:
rabbitmq-data:
+132 -6
View File
@@ -1,8 +1,8 @@
name: mtusi-messenger-lab4
name: mtusi-messenger-lab5
# Built once by the api service; migrate and s3-init consume the same image.
x-app: &app
image: mtusi-messenger:lab4
image: mtusi-messenger:lab5
services:
postgres:
@@ -23,7 +23,7 @@ services:
start_period: 5s
restart: unless-stopped
redis:
image: redis:7.4-alpine
image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
command:
- redis-server
- --appendonly
@@ -65,6 +65,26 @@ services:
start_period: 60s
restart: unless-stopped
broker:
image: rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528
hostname: broker
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-messenger}
RABBITMQ_DEFAULT_VHOST: ${BROKER_VHOST:-messenger}
volumes:
- rabbitmq-data:/var/lib/rabbitmq
ports:
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 12
start_period: 30s
restart: unless-stopped
s3-init:
<<: *app
entrypoint: ["/usr/local/bin/s3init"]
@@ -129,6 +149,19 @@ services:
S3_USE_PATH_STYLE: "true"
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1}
PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5}
PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s}
PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s}
PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s}
PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s}
PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s}
STAGING_ORPHAN_GRACE: ${STAGING_ORPHAN_GRACE:-1h}
STAGING_FAILED_RETENTION: ${STAGING_FAILED_RETENTION:-168h}
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
ports:
- "127.0.0.1:${SERVER_PORT:-8080}:8080"
depends_on:
@@ -148,10 +181,103 @@ services:
start_period: 5s
restart: unless-stopped
worker:
<<: *app
entrypoint: ["/usr/local/bin/worker"]
env_file:
- path: .env
required: false
environment:
APP_CONFIG: /etc/messenger/dev/config.yaml
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
POSTGRES_SSL_MODE: disable
S3_ENDPOINT: http://seaweedfs:8333
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret}
S3_USE_PATH_STYLE: "true"
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
WORKER_HEALTH_ADDR: ":8081"
FAULTS_ENABLED: ${FAULTS_ENABLED:-0}
FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-}
PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1}
PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5}
PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s}
PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s}
PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s}
PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s}
PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s}
ports:
- "127.0.0.1:${WORKER_HEALTH_PORT:-8081}:8081"
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8081/health/ready"]
interval: 10s
timeout: 5s
retries: 12
start_period: 10s
restart: unless-stopped
relay:
<<: *app
entrypoint: ["/usr/local/bin/relay"]
env_file:
- path: .env
required: false
environment:
APP_CONFIG: /etc/messenger/dev/config.yaml
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
POSTGRES_SSL_MODE: disable
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger}
RELAY_HEALTH_ADDR: ":8082"
FAULTS_ENABLED: ${FAULTS_ENABLED:-0}
FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-}
ports:
- "127.0.0.1:${RELAY_HEALTH_PORT:-8082}:8082"
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/health/ready"]
interval: 10s
timeout: 5s
retries: 12
start_period: 10s
restart: unless-stopped
volumes:
postgres18-data:
name: mtusi-messenger-lab4-postgres18-data
name: mtusi-messenger-lab5-postgres18-data
redis7-data:
name: mtusi-messenger-lab4-redis7-data
name: mtusi-messenger-lab5-redis7-data
seaweedfs-data:
name: mtusi-messenger-lab4-seaweedfs-data
name: mtusi-messenger-lab5-seaweedfs-data
rabbitmq-data:
name: mtusi-messenger-lab5-rabbitmq-data
+20
View File
@@ -0,0 +1,20 @@
# Lab5 evidence
All artifacts were produced against the local Compose stand on 2026-10-30
(2 vCPU, 2 GiB Docker VM, RabbitMQ 4.3.7, PostgreSQL 18, SeaweedFS 3.80).
- `stand.txt` — containers, image versions, VM limits.
- `smoke.txt` — public `make test`.
- `async.txt` — worker down/restart, broker restart with volume, stack restart.
- `variants.txt` — thumbnail/crop/optimized metadata, bytes, dimensions.
- `faults.txt` — permanent/transient retry, DLQ/replay, crash before/after
side effect, crash before/after ack, outbox gap, lease takeover.
- `persistence.txt`, `failover.txt`, `idempotency.txt`, `lb-readiness.txt` —
cumulative lab2–lab4 regressions.
- `track2/run.txt`, `track2/runs.csv`, `track2/summary.md` — track 2
duplicate/stale/version harness (3 repetitions).
- `unit-test.txt`, `integration-test.txt`, `vet.txt`, `lint.txt`,
`validate.txt` — build/quality gates.
No `.env`, tokens, credentials or presigned URLs are stored here. Raw SQL rows
contain only synthetic IDs and metadata.
+12
View File
@@ -0,0 +1,12 @@
# make test-queue 2026-10-31T11:26:40Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/lab5_async.py --base-url "http://localhost:8080"
worker down: 202 queued, content 409, queued attachment linked, API restarted
worker restarted: ready, thumbnail and original downloadable, staging cleared
broker down: upload accepted into outbox
broker + relay recovered: queued job reached ready
stack restart: ready attachment and original preserved
worker down: reprocess pipeline 2 accepted, v1 audit event recorded, upload still queued
worker up: reprocessed upload reached ready at pipeline 2, v1 superseded
worker down: reprocess pipeline 3 refused, v1 job untouched
worker up: preserved upload reached ready at pipeline 1
PASS: lab5 async worker/broker durability
+25
View File
@@ -0,0 +1,25 @@
# make test-attachments 2026-10-31T11:26:14Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/attachments_negative.py --base-url "http://localhost:8080"
10 MiB + 1 body: 413
empty body: 415
corrupt PNG (IDAT CRC): 415
MIME spoof (PNG declared as JPEG): 415
structurally valid PNG with non-zlib IDAT: 415
valid zlib, empty PNG raster: 415
valid zlib, invalid PNG scanline filter: 415
invalid PNG color type/bit depth pair: 415
duplicate PNG IHDR: 415
trailing bytes after PNG zlib stream: 415
tRNS on RGBA PNG: 415
short tRNS on RGB PNG: 415
PLTE entries exceed palette bit depth: 415
palette tRNS exceeds PLTE entries: 415
dangerous filenames: 400
foreign attachment metadata/content: 404
empty text without attachments: 400
attachment from another chat: 404
11 attachment ids: 400
duplicate attachment ids: 400
attachment-only message: 201
anonymous direct S3 GET: 403
PASS: lab4 attachment negative scenarios
+29
View File
@@ -0,0 +1,29 @@
# db state 2026-10-30T20:16:08Z
status | count
---------+-------
failed | 2
pending | 4
ready | 31
(3 rows)
status | count
--------+-------
failed | 2
ready | 34
(2 rows)
variants
----------
102
(1 row)
unpublished | outbox
-------------+--------
0 | 58
(1 row)
Timeout: 60.0 seconds ...
Listing queues for vhost messenger ...
name messages messages_ready messages_unacknowledged consumers
attachment.process.dead 4 4 0 0
attachment.process.v1 0 0 0 0
+5
View File
@@ -0,0 +1,5 @@
# failover 2026-10-30T20:11:43Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 0, "first_recovery_seconds": 0.221}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
# make test-faults 2026-10-31T11:27:03Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/lab5_faults.py --base-url "http://localhost:8080"
permanent failure: failed/injected_failure, content 409, failed attachment not linkable, registry visible
replay: failed -> queued -> ready, one thumbnail
replay ready job: refused
replay older failed version while v2 queued: refused, state intact
v2 published after older replay refusal
replay of failed reprocess: queued projection restored, then ready at pipeline 2
transient exhaustion: attempts=5, retry_exhausted, four retries, DLQ message present
crash after first variant: redelivered, same three variants, attempts=2
crash after commit before ack: ready remains, redelivery is a no-op
crash after claim before side effect: requeued, attempts=2, ready
outbox gap: crash before publish recovered through the relay, outbox published, ready
lease expiry: paused worker, reaper requeued, stale attempt fenced, ready
PASS: lab5 level 4 fault injection
+4
View File
@@ -0,0 +1,4 @@
# idempotency 2026-10-30T20:11:43Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/idempotency_parallel.py --base-url "http://localhost:8080"
{"replay": {"workers": 12, "instances": ["1c67613c89f2", "ee96975adb44"], "message_id": "01a18e70-8fcd-7519-91ab-d9c43b6a5f37"}, "conflict": {"workers": 12, "accepted": 1, "conflicts": 11}, "stable_body": {"message_id": "01a18e70-8ff9-759f-ab87-abd58f58c59d"}, "key_required": {"status": 400}}
PASS: параллельный повтор с одним Idempotency-Key создаёт ровно одно сообщение
+2
View File
@@ -0,0 +1,2 @@
ok mtusi-messenger/internal/storage/postgres 9.448s
ok mtusi-messenger/internal/storage/redis 1.238s
+31
View File
@@ -0,0 +1,31 @@
# LB excludes a not-ready but reachable replica (lab5)
# generated: 2026-10-30T20:18:32Z
# lb: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
## 1. baseline: requests through LB by replica
22 23d07355bd77
18 56bd7451c245
stats: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias
instance-id: 42c73eb4c0a5, readiness while its database is reachable: 200
load balancer slot for the candidate: api2, stats: api1=UP api2=UP 1/2 api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 3. cut the candidate's database access (stop only the TCP proxy)
candidate is still running: running, readiness: 503
waiting for the load balancer to mark api2 DOWN
api2 went DOWN 1s after its database was cut (check: inter 2s fall 2)
stats after failure: api1=UP api2=DOWN api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 4. 40 requests through LB while the candidate is not ready
22 23d07355bd77
18 56bd7451c245
(the candidate must not appear above)
## 5. business writes through LB during the exclusion
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 816.58,
## 6. cleanup
stats after cleanup: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
+34
View File
@@ -0,0 +1,34 @@
# lb readiness 2026-10-30T20:18:31Z
scripts/lb_readiness_check.sh
# LB excludes a not-ready but reachable replica (lab5)
# generated: 2026-10-30T20:18:32Z
# lb: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
## 1. baseline: requests through LB by replica
22 23d07355bd77
18 56bd7451c245
stats: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias
instance-id: 42c73eb4c0a5, readiness while its database is reachable: 200
load balancer slot for the candidate: api2, stats: api1=UP api2=UP 1/2 api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 3. cut the candidate's database access (stop only the TCP proxy)
candidate is still running: running, readiness: 503
waiting for the load balancer to mark api2 DOWN
api2 went DOWN 1s after its database was cut (check: inter 2s fall 2)
stats after failure: api1=UP api2=DOWN api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 4. 40 requests through LB while the candidate is not ready
22 23d07355bd77
18 56bd7451c245
(the candidate must not appear above)
## 5. business writes through LB during the exclusion
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 816.58,
## 6. cleanup
stats after cleanup: api1=UP api2=MAINT (resolution) api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
lb_readiness_check.sh: evidence in evidence/lab5/lb-readiness-detail.txt
+20
View File
@@ -0,0 +1,20 @@
/Users/vladislav/Projects/mtusi2026lab5/bin/golangci-lint run -v
level=info msg="golangci-lint has version 2.13.2 built with go1.27.1 from (unknown, modified: ?, mod sum: \"h1:bCyq3E4vo9qwzifjpzJqYndEt7Ncva80qkk8G2B4TXU=\") on (unknown)"
level=info msg="[config_reader] Config search paths: [./ /Users/vladislav/Projects/mtusi2026lab5 /Users/vladislav/Projects /Users/vladislav /Users /]"
level=info msg="[config_reader] Used config file .golangci.yml"
level=info msg="[config_reader] Module name \"mtusi-messenger\""
level=info msg="[goenv] Read go env for 6.264542ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}"
level=info msg="[lintersdb] Active 50 linters: [asasalint bodyclose canonicalheader containedctx contextcheck copyloopvar depguard durationcheck errcheck errname errorlint exhaustive exptostd fatcontext forbidigo forcetypeassert gocheckcompilerdirectives gocritic gofumpt goimports gomoddirectives gosec govet ineffassign intrange makezero misspell musttag nilerr nilnil noctx nolintlint perfsprint prealloc reassign recvcheck revive rowserrcheck sloglint sqlclosecheck staticcheck testifylint thelper tparallel unconvert unparam unused usestdlibvars usetesting wastedassign]"
level=info msg="[loader] Using build tags: [integration]"
level=info msg="[loader] Go packages loading at mode 8767 (compiled_files|deps|exports_file|files|types_sizes|imports|name) took 193.808375ms"
level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 11.667ms"
level=info msg="[linters_context/goanalysis] analyzers took 0s with no stages"
level=info msg="[runner/exclusion_rules] Skipped 14 issues by rules: [Path: \"_test\\\\.go\", Linters: \"gosec, prealloc, forcetypeassert, unparam\"]"
level=info msg="[runner] Issues before processing: 887, after processing: 0"
level=info msg="[runner] Processors filtering stat (in/out): path_absoluter: 887/887, cgo: 887/887, filename_unadjuster: 887/887, nolint_filter: 6/0, invalid_issue: 887/887, exclusion_paths: 887/887, path_relativity: 887/887, generated_file_filter: 887/298, exclusion_rules: 298/6"
level=info msg="[runner] processing took 2.828168ms with stages: generated_file_filter: 1.490334ms, nolint_filter: 764.209µs, path_relativity: 284.917µs, exclusion_rules: 251.001µs, invalid_issue: 13.417µs, cgo: 12.25µs, path_absoluter: 4.666µs, sort_results: 2.916µs, filename_unadjuster: 2.542µs, max_same_issues: 874ns, diff: 167ns, fixer: 166ns, path_prettifier: 125ns, max_per_file_from_linter: 125ns, exclusion_paths: 125ns, path_shortener: 124ns, uniq_by_line: 84ns, severity-rules: 42ns, max_from_linter: 42ns, source_code: 42ns"
level=info msg="[runner] linters took 629.148708ms with stages: goanalysis_metalinter: 626.277292ms"
0 issues.
level=info msg="File cache stats: 0 entries of total size 0B"
level=info msg="Memory: 10 samples, avg is 87.2MB, max is 111.5MB"
level=info msg="Execution took 841.622625ms"
+4
View File
@@ -0,0 +1,4 @@
# persistence 2026-10-30T20:12:05Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+11
View File
@@ -0,0 +1,11 @@
# presign 2026-10-30T20:11:41Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 student-tests/presign_flow.py --base-url "http://localhost:8080"
happy path: direct PUT 200, complete queued, ready after worker
message with attachment downloaded as the second chat member
repeated complete is idempotent
foreign complete: 404
complete before PUT: 409
wrong completion checksum: 409
oversized direct PUT: 403 from S3; complete: 409 (nothing stored)
same-size overwrite: last PUT wins, checksum verified
PASS: lab4 presigned upload flow
+3
View File
@@ -0,0 +1,3 @@
# make test 2026-10-31T11:26:59Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+20
View File
@@ -0,0 +1,20 @@
# stand 2026-10-30T20:16:07Z
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp
mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp
mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 8 minutes ago Up 8 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp
mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 8 minutes ago Up 3 minutes 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp
mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 8 minutes ago Up 3 minutes (healthy) 5432/tcp
mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 8 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8082->8082/tcp
mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 8 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp
mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8081->8081/tcp
mtusi-messenger:lab5 10b83ff52c7d
rabbitmq:4.3.7-management-alpine 1528a88eebf2
haproxy:3.2-alpine 254e6370f01c
redis:7.4-alpine 858f009f9709
postgres:18-alpine 77f585114c32
chrislusf/seaweedfs:3.80 1055999e08ee
CPUs=2 MemTotal=2053644288
+7
View File
@@ -0,0 +1,7 @@
Dduplicate_before_effect rep=1: PASS
Dstale_version_after_v2 rep=1: PASS
Dduplicate_before_effect rep=2: PASS
Dstale_version_after_v2 rep=2: PASS
Dduplicate_before_effect rep=3: PASS
Dstale_version_after_v2 rep=3: PASS
PASS: track2 6/6 scenarios
+7
View File
@@ -0,0 +1,7 @@
scenario,rep,pass,detail
duplicate_before_effect,1,1,attempts=1
stale_version_after_v2,1,1,v1_job=01a18e6e-17b0-763b-b330-7adbcb43a700
duplicate_before_effect,2,1,attempts=1
stale_version_after_v2,2,1,v1_job=01a18e6e-455e-75f3-857a-750ed660378b
duplicate_before_effect,3,1,attempts=1
stale_version_after_v2,3,1,v1_job=01a18e6e-74f2-7b4a-842b-c014c626413c
1 scenario rep pass detail
2 duplicate_before_effect 1 1 attempts=1
3 stale_version_after_v2 1 1 v1_job=01a18e6e-17b0-763b-b330-7adbcb43a700
4 duplicate_before_effect 2 1 attempts=1
5 stale_version_after_v2 2 1 v1_job=01a18e6e-455e-75f3-857a-750ed660378b
6 duplicate_before_effect 3 1 attempts=1
7 stale_version_after_v2 3 1 v1_job=01a18e6e-74f2-7b4a-842b-c014c626413c
+21
View File
@@ -0,0 +1,21 @@
# Track 2 — versioned pipeline and duplicate/stale delivery
Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`,
deterministic object keys, a serving version that changes only when all v2
variants are committed, and stale-version acknowledgements, an at-least-once
pipeline loses no accepted work and never overwrites a newer result with an
older duplicate.
Method: N=3 repetitions. Each repetition creates fresh users/chat and runs:
- duplicate pipeline publication while the worker is stopped, followed by recovery;
- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready.
Invariants checked: unique variant rows per effect key; ready remains ready;
serving pipeline version never regresses; original bytes/checksum unchanged;
stale job state is not rewritten.
Limitations: this harness covers duplicate and stale/reorder delivery. Crash
windows (before/after side effect, before/after ack, outbox gap) and lease
takeover are covered by student-tests/lab5_faults.py; both are single-node
Compose evidence, not a production availability claim. Delivery is
at-least-once; exactly-once is not claimed.
+41
View File
@@ -0,0 +1,41 @@
env -u APP_ENV -u SERVER_HOST -u SERVER_PORT -u SERVER_READ_HEADER_TIMEOUT -u SERVER_READ_TIMEOUT -u SERVER_SHUTDOWN_TIMEOUT -u SERVER_MAX_BODY_BYTES -u LOG_LEVEL -u LOG_FORMAT -u POSTGRES_HOST -u POSTGRES_PORT -u POSTGRES_DATABASE -u POSTGRES_USER -u POSTGRES_PASSWORD -u POSTGRES_SSL_MODE -u POSTGRES_MAX_CONNS -u POSTGRES_MIN_CONNS -u POSTGRES_MAX_CONN_LIFETIME -u POSTGRES_MAX_CONN_IDLE_TIME -u POSTGRES_CONNECT_TIMEOUT -u POSTGRES_QUERY_TIMEOUT -u REDIS_ENABLED -u REDIS_HOST -u REDIS_PORT -u REDIS_PASSWORD -u REDIS_DB -u REDIS_MAX_RETRIES -u REDIS_DIAL_TIMEOUT -u REDIS_READ_TIMEOUT -u REDIS_WRITE_TIMEOUT -u REDIS_POOL_SIZE -u CACHE_ENABLED -u CACHE_CHAT_LIST_TTL -u CACHE_LOCK_TTL -u CACHE_LOCK_WAIT_TIMEOUT -u SESSION_TTL_SECONDS -u IDEMPOTENCY_TTL_SECONDS -u API_REPLICAS -u API_CPU_LIMIT -u API_MEM_LIMIT -u POSTGRES_MAX_CONNS -u POSTGRES_CPU_LIMIT -u POSTGRES_MEM_LIMIT -u LB_STATS_PORT -u S3_ENDPOINT -u S3_PUBLIC_ENDPOINT -u S3_PORT -u S3_REGION -u S3_BUCKET -u S3_ACCESS_KEY_ID -u S3_SECRET_ACCESS_KEY -u MAX_UPLOAD_BYTES -u MAX_IMAGE_PIXELS -u S3_PRESIGN_TTL -u S3_SWEEP_INTERVAL -u S3_SWEEP_GRACE -u BROKER_USER -u BROKER_PASSWORD -u BROKER_VHOST -u BROKER_URL -u BROKER_EXCHANGE -u BROKER_QUEUE -u BROKER_DLX -u BROKER_DLQ -u BROKER_MANAGEMENT_PORT -u BROKER_CPU_LIMIT -u BROKER_MEM_LIMIT -u WORKER_CONCURRENCY -u WORKER_PREFETCH -u WORKER_CPU_LIMIT -u WORKER_MEM_LIMIT -u WORKER_SHUTDOWN_TIMEOUT -u PROCESSING_PIPELINE_VERSION -u PROCESSING_MAX_ATTEMPTS -u PROCESSING_JOB_TIMEOUT -u PROCESSING_LEASE_TTL -u PROCESSING_HEARTBEAT -u PROCESSING_RETRY_BASE -u PROCESSING_RETRY_CAP -u OUTBOX_POLL_INTERVAL -u OUTBOX_CLAIM_TTL -u OUTBOX_BATCH -u OUTBOX_REAPER_INTERVAL -u OUTBOX_RETENTION -u RELAY_CPU_LIMIT -u RELAY_MEM_LIMIT -u STAGING_ORPHAN_GRACE -u STAGING_FAILED_RETENTION -u ADMISSION_MAX_PENDING_PER_USER -u ADMISSION_MAX_PENDING -u ADMISSION_MAX_STAGED_BYTES go test -race -shuffle=on -count=1 -timeout=5m ./...
? mtusi-messenger/cmd/jobctl [no test files]
? mtusi-messenger/cmd/messenger [no test files]
? mtusi-messenger/cmd/migrate [no test files]
? mtusi-messenger/cmd/relay [no test files]
? mtusi-messenger/cmd/s3init [no test files]
? mtusi-messenger/cmd/worker [no test files]
? mtusi-messenger/internal/broker [no test files]
ok mtusi-messenger/internal/config 1.717s
ok mtusi-messenger/internal/domain 2.647s
? mtusi-messenger/internal/failpoint [no test files]
ok mtusi-messenger/internal/health 1.315s
ok mtusi-messenger/internal/imagecheck 3.766s
ok mtusi-messenger/internal/imageproc 2.400s
ok mtusi-messenger/internal/logger 2.338s
? mtusi-messenger/internal/outbox [no test files]
ok mtusi-messenger/internal/platform/cache 4.909s
? mtusi-messenger/internal/platform/cache/mocks [no test files]
ok mtusi-messenger/internal/platform/objectstore 6.507s
ok mtusi-messenger/internal/platform/password 3.349s
ok mtusi-messenger/internal/platform/token 3.304s
ok mtusi-messenger/internal/service/attachment 5.105s
? mtusi-messenger/internal/service/attachment/mocks [no test files]
ok mtusi-messenger/internal/service/chat 4.460s
? mtusi-messenger/internal/service/chat/mocks [no test files]
ok mtusi-messenger/internal/service/message 4.119s
? mtusi-messenger/internal/service/message/mocks [no test files]
ok mtusi-messenger/internal/service/session 5.771s
? mtusi-messenger/internal/service/session/mocks [no test files]
ok mtusi-messenger/internal/service/user 6.251s
? mtusi-messenger/internal/service/user/mocks [no test files]
ok mtusi-messenger/internal/storage/postgres 6.484s
? mtusi-messenger/internal/storage/postgres/internal/sqlcgen [no test files]
? mtusi-messenger/internal/storage/postgres/migrations [no test files]
? mtusi-messenger/internal/storage/redis [no test files]
? mtusi-messenger/internal/storage/redis/mocks [no test files]
ok mtusi-messenger/internal/transport/rest 6.893s
? mtusi-messenger/internal/transport/rest/internal/oas [no test files]
? mtusi-messenger/internal/transport/rest/mocks [no test files]
? mtusi-messenger/internal/worker [no test files]
? mtusi-messenger/tools/specgen [no test files]
+3
View File
@@ -0,0 +1,3 @@
# make validate PYTHON=.venv/bin/python 2026-10-30T22:22:13Z
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+3
View File
@@ -0,0 +1,3 @@
# lab5_variants 2026-10-30T20:08:36Z
variants: thumbnail 256x160, crop 128x128, optimized WebP 320x200; metadata == bytes
PASS: lab5 variant contract
View File
+8 -8
View File
@@ -3,23 +3,23 @@
# lb: haproxy:3.2-alpine
## 1. baseline: requests through LB by replica
18 52861386fddb
22 8587b4251b72
stats: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
18 52861386fddb
22 8587b4251b72
stats: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias
instance-id: 9e9e905440d3, readiness while its database is reachable: 200
load balancer slot for the candidate: api3, stats: api1=UP api2=UP api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
load balancer slot for the candidate: api3, stats: api1=UP api2=UP api3=UP api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 3. cut the candidate's database access (stop only the TCP proxy)
candidate is still running: running, readiness: 503
waiting for the load balancer to mark api3 DOWN
api3 went DOWN 2s after its database was cut (check: inter 2s fall 2)
stats after failure: api1=UP api2=UP api3=DOWN api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
stats after failure: api1=UP api2=UP api3=DOWN api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
## 4. 40 requests through LB while the candidate is not ready
18 52861386fddb
22 8587b4251b72
18 52861386fddb
22 8587b4251b72
(the candidate must not appear above)
## 5. business writes through LB during the exclusion
@@ -28,4 +28,4 @@
"successful_rps": 1163.529,
## 6. cleanup
stats after cleanup: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
stats after cleanup: api1=UP api2=UP api3=MAINT (resolution) api4=MAINT (resolution) api5=MAINT (resolution) api6=MAINT (resolution) api7=MAINT (resolution) api8=MAINT (resolution) BACKEND=UP
+9 -5
View File
@@ -7,6 +7,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4
github.com/aws/smithy-go v1.28.6
github.com/gen2brain/webp v0.6.4
github.com/go-faster/errors v0.8.0
github.com/go-faster/jx v1.2.0
github.com/go-playground/validator/v10 v10.30.5
@@ -14,13 +15,15 @@ require (
github.com/jackc/pgx/v5 v5.11.0
github.com/ogen-go/ogen v1.24.0
github.com/pressly/goose/v3 v3.28.0
github.com/rabbitmq/amqp091-go v1.16.0
github.com/redis/go-redis/v9 v9.22.0
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.12.1
go.uber.org/zap v1.28.0
go.uber.org/zap/exp v0.3.0
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
golang.org/x/image v0.47.0
golang.org/x/sync v0.24.0
)
require (
@@ -42,6 +45,7 @@ require (
github.com/cubicdaiya/gonp v1.0.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dlclark/regexp2 v1.12.0 // indirect
github.com/ebitengine/purego v0.10.1 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/fatih/structtag v1.2.0 // indirect
@@ -111,11 +115,11 @@ require (
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/net v0.59.0 // indirect
golang.org/x/sys v0.49.0 // indirect
golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/tools v0.49.0 // indirect
golang.org/x/text v0.43.0 // indirect
golang.org/x/tools v0.51.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a // indirect
google.golang.org/grpc v1.83.2 // indirect
+18 -10
View File
@@ -54,6 +54,8 @@ github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz
github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
@@ -66,6 +68,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU=
github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-faster/errors v0.8.0 h1:9T9eJrM+72dFk7n4DfhuaDDe6cyuFCSW2oNUkN77Yqc=
@@ -181,6 +185,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.28.0 h1:D2M+iL31GmpZxSHOhX8mqyqAT3CXnokUmm0eKoSP+Vc=
github.com/pressly/goose/v3 v3.28.0/go.mod h1:v26MOuB8bL3kzzrt3Vqhb3R0PRVsl8hFQKdrht/L6Rk=
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0=
github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
@@ -284,32 +290,34 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+320
View File
@@ -0,0 +1,320 @@
// Package broker owns the RabbitMQ topology, confirmed publishing and the
// consumer loop. Business state never lives here: PostgreSQL is the source of
// truth for jobs and the outbox.
package broker
import (
"context"
"errors"
"fmt"
"sync"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"mtusi-messenger/internal/domain"
)
type Config struct {
URL string
Exchange string
DLX string
Queue string
DLQ string
ConnectTimeout time.Duration
ConfirmTimeout time.Duration
Heartbeat time.Duration
Prefetch int
}
type Client struct {
cfg Config
conn *amqp.Connection
publisher *amqp.Channel
mu sync.Mutex
returns chan amqp.Return
closed chan *amqp.Error
}
// Dial connects, declares the durable topology idempotently and opens a
// confirm-mode publisher channel.
func Dial(cfg Config) (*Client, error) {
if cfg.URL == "" {
return nil, errors.New("broker url is empty")
}
conn, err := amqp.DialConfig(cfg.URL, amqp.Config{
Heartbeat: cfg.Heartbeat,
Dial: amqp.DefaultDial(cfg.ConnectTimeout),
Properties: amqp.Table{
"connection_name": "messenger-relay",
},
})
if err != nil {
return nil, fmt.Errorf("dial broker: %w", err)
}
channel, err := conn.Channel()
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("open topology channel: %w", err)
}
if err := declareTopology(channel, cfg); err != nil {
_ = channel.Close()
_ = conn.Close()
return nil, err
}
if err := channel.Close(); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("close topology channel: %w", err)
}
publisher, err := conn.Channel()
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("open publisher channel: %w", err)
}
client := &Client{
cfg: cfg,
conn: conn,
publisher: publisher,
returns: make(chan amqp.Return, 16),
closed: conn.NotifyClose(make(chan *amqp.Error, 1)),
}
if err := publisher.Confirm(false); err != nil {
_ = client.Close()
return nil, fmt.Errorf("enable publisher confirms: %w", err)
}
publisher.NotifyReturn(client.returns)
return client, nil
}
func declareTopology(channel *amqp.Channel, cfg Config) error {
if err := channel.ExchangeDeclare(cfg.Exchange, "direct", true, false, false, false, nil); err != nil {
return fmt.Errorf("declare exchange: %w", err)
}
if err := channel.ExchangeDeclare(cfg.DLX, "direct", true, false, false, false, nil); err != nil {
return fmt.Errorf("declare dlx: %w", err)
}
queue, err := channel.QueueDeclare(cfg.Queue, true, false, false, false, amqp.Table{
"x-queue-type": "quorum",
"x-delivery-limit": int32(100),
"x-delayed-retry-type": "all",
"x-delayed-retry-min": int32(5000),
"x-delayed-retry-max": int32(60000),
"x-dead-letter-exchange": cfg.DLX,
"x-dead-letter-routing-key": cfg.DLQ,
"x-dead-letter-strategy": "at-least-once",
"x-overflow": "reject-publish",
"x-max-length": int32(100000),
"x-consumer-timeout": int32(300000),
})
if err != nil {
return fmt.Errorf("declare job queue: %w", err)
}
if queue.Name != cfg.Queue {
return fmt.Errorf("broker returned queue %q, want %q", queue.Name, cfg.Queue)
}
if err := channel.QueueBind(cfg.Queue, cfg.Queue, cfg.Exchange, false, nil); err != nil {
return fmt.Errorf("bind job queue: %w", err)
}
if _, err := channel.QueueDeclare(cfg.DLQ, true, false, false, false, amqp.Table{
"x-queue-type": "quorum",
"x-max-length": int32(100000),
"x-overflow": "reject-publish",
"x-message-ttl": int32(7 * 24 * 60 * 60 * 1000),
}); err != nil {
return fmt.Errorf("declare dlq: %w", err)
}
if err := channel.QueueBind(cfg.DLQ, cfg.DLQ, cfg.DLX, false, nil); err != nil {
return fmt.Errorf("bind dlq: %w", err)
}
return nil
}
func (c *Client) Connected() bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.conn != nil && !c.conn.IsClosed()
}
// PublishConfirmed returns only after the broker acknowledged the persistent
// message. A duplicate publication after a crash is normal and harmless.
func (c *Client) PublishConfirmed(ctx context.Context, routingKey string, message domain.JobMessage) error {
payload, err := message.MarshalPayload()
if err != nil {
return fmt.Errorf("marshal job message: %w", err)
}
confirmation, err := c.publisher.PublishWithDeferredConfirmWithContext(
ctx,
c.cfg.Exchange,
routingKey,
true,
false,
amqp.Publishing{
ContentType: "application/json",
DeliveryMode: amqp.Persistent,
MessageId: message.JobID.String(),
Type: message.Type,
Timestamp: message.CreatedAt,
Body: payload,
},
)
if err != nil {
return fmt.Errorf("publish job: %w", err)
}
confirmCtx := ctx
if c.cfg.ConfirmTimeout > 0 {
var cancel context.CancelFunc
confirmCtx, cancel = context.WithTimeout(ctx, c.cfg.ConfirmTimeout)
defer cancel()
}
confirmed, err := confirmation.WaitContext(confirmCtx)
if err != nil {
return fmt.Errorf("await publisher confirm: %w", err)
}
if !confirmed {
return errors.New("broker nacked the job message")
}
// mandatory=true: an unroutable message is returned even though the
// confirm may still be positive. The topology is declared before publish,
// so this is a safety net rather than a normal path.
select {
case returned := <-c.returns:
if returned.MessageId == message.JobID.String() {
return fmt.Errorf("broker returned unroutable message: %s", returned.ReplyText)
}
default:
}
return nil
}
type Outcome int
const (
Ack Outcome = iota
Requeue
Drop
)
type Handler func(ctx context.Context, delivery amqp.Delivery) Outcome
// Consume blocks until the context is cancelled or the consumer channel fails.
// Each call uses its own channel so worker slots do not share prefetch state.
func (c *Client) Consume(ctx context.Context, handler Handler) error {
channel, err := c.conn.Channel()
if err != nil {
return fmt.Errorf("open consumer channel: %w", err)
}
defer channel.Close()
if err := channel.Qos(c.cfg.Prefetch, 0, false); err != nil {
return fmt.Errorf("set qos: %w", err)
}
consumerTag := fmt.Sprintf("messenger-%d", time.Now().UnixNano())
deliveries, err := channel.Consume(
c.cfg.Queue,
consumerTag,
false,
false,
false,
false,
nil,
)
if err != nil {
return fmt.Errorf("consume: %w", err)
}
done := make(chan struct{})
defer close(done)
go func() {
select {
case <-ctx.Done():
_ = channel.Cancel(consumerTag, false)
case <-done:
}
}()
for delivery := range deliveries {
outcome := safeHandle(ctx, handler, delivery)
switch outcome {
case Ack:
_ = delivery.Ack(false)
case Requeue:
_ = delivery.Nack(false, true)
case Drop:
_ = delivery.Nack(false, false)
}
if ctx.Err() != nil {
// Let unacked deliveries return to the broker on channel close.
return nil
}
}
if ctx.Err() != nil {
return nil
}
return errors.New("consumer channel closed")
}
func safeHandle(ctx context.Context, handler Handler, delivery amqp.Delivery) (outcome Outcome) {
defer func() {
if recover() != nil {
outcome = Requeue
}
}()
return handler(ctx, delivery)
}
func (c *Client) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.publisher != nil {
_ = c.publisher.Close()
c.publisher = nil
}
if c.conn != nil {
err := c.conn.Close()
c.conn = nil
return err
}
return nil
}
+123
View File
@@ -37,6 +37,75 @@ type Config struct {
Session Session `mapstructure:"session"`
Idempotency Idempotency `mapstructure:"idempotency"`
S3 S3 `mapstructure:"s3"`
Broker Broker `mapstructure:"broker"`
Worker Worker `mapstructure:"worker"`
Relay Relay `mapstructure:"relay"`
Outbox Outbox `mapstructure:"outbox"`
Processing Processing `mapstructure:"processing"`
Staging Staging `mapstructure:"staging"`
Admission Admission `mapstructure:"admission"`
}
// Broker carries the AMQP transport settings. The API never publishes; relay
// and worker require a non-empty URL at startup.
type Broker struct {
URL string `mapstructure:"url"`
ConnectTimeout time.Duration `mapstructure:"connect_timeout"`
ConfirmTimeout time.Duration `mapstructure:"confirm_timeout"`
Heartbeat time.Duration `mapstructure:"heartbeat"`
Exchange string `mapstructure:"exchange" validate:"required"`
DLX string `mapstructure:"dlx" validate:"required"`
Queue string `mapstructure:"queue" validate:"required"`
DLQ string `mapstructure:"dlq" validate:"required"`
Prefetch int `mapstructure:"prefetch" validate:"required,gt=0"`
}
// Worker bounds the process that consumes jobs.
type Worker struct {
Concurrency int `mapstructure:"concurrency" validate:"required,gt=0,lte=32"`
Prefetch int `mapstructure:"prefetch" validate:"required,gt=0,lte=100"`
ShutdownTimeout time.Duration `mapstructure:"shutdown_timeout" validate:"required,gt=0"`
HealthAddr string `mapstructure:"health_addr" validate:"required"`
}
// Relay bounds the outbox publisher and lease reaper.
type Relay struct {
HealthAddr string `mapstructure:"health_addr" validate:"required"`
}
// Outbox schedules publications and reclaims abandoned relay claims.
type Outbox struct {
PollInterval time.Duration `mapstructure:"poll_interval" validate:"required,gt=0"`
ClaimTTL time.Duration `mapstructure:"claim_ttl" validate:"required,gt=0"`
Batch int `mapstructure:"batch" validate:"required,gt=0,lte=1000"`
ReaperInterval time.Duration `mapstructure:"reaper_interval" validate:"required,gt=0"`
ReconcileAfter time.Duration `mapstructure:"reconcile_after" validate:"omitempty,gt=0"`
Retention time.Duration `mapstructure:"retention" validate:"required,gt=0"`
}
// Processing holds the shared job contract knobs read by API and worker.
type Processing struct {
PipelineVersion int `mapstructure:"pipeline_version" validate:"required,gte=1,lte=2"`
MaxAttempts int `mapstructure:"max_attempts" validate:"required,gte=1,lte=100"`
JobTimeout time.Duration `mapstructure:"job_timeout" validate:"required,gt=0"`
LeaseTTL time.Duration `mapstructure:"lease_ttl" validate:"required,gt=0"`
Heartbeat time.Duration `mapstructure:"heartbeat" validate:"required,gt=0"`
RetryBase time.Duration `mapstructure:"retry_base" validate:"required,gt=0"`
RetryCap time.Duration `mapstructure:"retry_cap" validate:"required,gt=0"`
}
// Staging controls cleanup of objects that are no longer the active source.
type Staging struct {
OrphanGrace time.Duration `mapstructure:"orphan_grace" validate:"required,gt=0"`
FailedRetention time.Duration `mapstructure:"failed_retention" validate:"required,gt=0"`
}
// Admission bounds accepted-but-unfinished work per user and globally.
type Admission struct {
MaxPendingPerUser int `mapstructure:"max_pending_per_user" validate:"required,gte=0"`
MaxPending int `mapstructure:"max_pending" validate:"required,gte=0"`
MaxStagedBytes int64 `mapstructure:"max_staged_bytes" validate:"required,gte=0"`
MaxConcurrent int `mapstructure:"max_concurrent_uploads" validate:"omitempty,gt=0"`
}
// S3 describes the object store and attachment limits. Access keys are checked
@@ -178,6 +247,38 @@ var defaults = map[string]any{
"cache.chat_list_ttl": "5m",
"cache.lock_ttl": "10s",
"cache.lock_wait_timeout": "2s",
"broker.url": "",
"broker.connect_timeout": "5s",
"broker.confirm_timeout": "5s",
"broker.heartbeat": "10s",
"broker.exchange": "attachments.jobs",
"broker.dlx": "attachments.dlx",
"broker.queue": "attachment.process.v1",
"broker.dlq": "attachment.process.dead",
"broker.prefetch": 1,
"worker.concurrency": 1,
"worker.prefetch": 1,
"worker.shutdown_timeout": "30s",
"worker.health_addr": ":8081",
"relay.health_addr": ":8082",
"outbox.poll_interval": "1s",
"outbox.claim_ttl": "30s",
"outbox.batch": 20,
"outbox.reaper_interval": "15s",
"outbox.retention": "168h",
"processing.pipeline_version": 1,
"processing.max_attempts": 5,
"processing.job_timeout": "120s",
"processing.lease_ttl": "180s",
"processing.heartbeat": "30s",
"processing.retry_base": "2s",
"processing.retry_cap": "60s",
"staging.orphan_grace": "1h",
"staging.failed_retention": "168h",
"admission.max_pending_per_user": 20,
"admission.max_pending": 200,
"admission.max_staged_bytes": 2147483648,
}
func Load() (Config, error) {
@@ -233,6 +334,28 @@ func (c Config) validateDependencies() error {
return errors.New("validate config: s3.sweep_grace must exceed s3.presign_ttl")
}
if c.Processing.LeaseTTL < 3*c.Processing.Heartbeat {
return errors.New("validate config: processing.lease_ttl must be at least 3x processing.heartbeat")
}
if c.Processing.JobTimeout >= c.Processing.LeaseTTL {
return errors.New("validate config: processing.job_timeout must be below processing.lease_ttl")
}
if c.Processing.RetryCap < c.Processing.RetryBase {
return errors.New("validate config: processing.retry_cap must be >= processing.retry_base")
}
if c.Staging.FailedRetention < c.Staging.OrphanGrace {
return errors.New("validate config: staging.failed_retention must be >= staging.orphan_grace")
}
if c.Broker.URL != "" {
if _, err := url.Parse(c.Broker.URL); err != nil {
return fmt.Errorf("validate config: broker.url: %w", err)
}
}
return nil
}
+107 -26
View File
@@ -6,53 +6,127 @@ import (
"github.com/google/uuid"
)
// AttachmentStatus is the lifecycle of an attachment. Only ready attachments
// are exposed to members; pending rows exist while the object is being written,
// failed rows are reserved for later labs.
// AttachmentStatus is the public lifecycle of an attachment. Pending is an
// internal presign state and is never exposed by the API.
type AttachmentStatus string
const (
AttachmentPending AttachmentStatus = "pending"
AttachmentReady AttachmentStatus = "ready"
AttachmentFailed AttachmentStatus = "failed"
AttachmentPending AttachmentStatus = "pending"
AttachmentQueued AttachmentStatus = "queued"
AttachmentProcessing AttachmentStatus = "processing"
AttachmentReady AttachmentStatus = "ready"
AttachmentFailed AttachmentStatus = "failed"
)
func (s AttachmentStatus) Valid() bool {
switch s {
case AttachmentPending, AttachmentReady, AttachmentFailed:
case AttachmentPending, AttachmentQueued, AttachmentProcessing, AttachmentReady, AttachmentFailed:
return true
default:
return false
}
}
// Attachment describes the original object. ObjectKey is internal: it never
// leaves the service and must not be mapped into an API response.
type Attachment struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
Filename string
// Public reports whether the status is part of the API enum.
func (s AttachmentStatus) Public() bool {
switch s {
case AttachmentQueued, AttachmentProcessing, AttachmentReady, AttachmentFailed:
return true
default:
return false
}
}
// VariantName is the contract variant identifier.
type VariantName string
const (
VariantOriginal VariantName = "original"
VariantThumbnail VariantName = "thumbnail"
VariantCrop VariantName = "crop"
VariantOptimized VariantName = "optimized"
)
func (n VariantName) Valid() bool {
switch n {
case VariantThumbnail, VariantCrop, VariantOptimized:
return true
default:
return false
}
}
// ContentType returns the contract media type of the variant.
func (n VariantName) ContentType() string {
if n == VariantOptimized {
return "image/webp"
}
return "image/jpeg"
}
// ObjectName returns the object suffix for the variant.
func (n VariantName) ObjectName() string {
if n == VariantOptimized {
return string(n) + ".webp"
}
return string(n) + ".jpg"
}
// Variant is one derived image. ObjectKey is internal and never leaves the
// service.
type Variant struct {
Name VariantName
ContentType string
SizeBytes int64
Width int
Height int
SHA256 string
Status AttachmentStatus
ErrorCode *string
ObjectKey string
CreatedAt time.Time
UpdatedAt time.Time
}
// Attachment describes the original object and the current pipeline result.
type Attachment struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
Filename string
ContentType string
SizeBytes int64
SHA256 string
Status AttachmentStatus
ErrorCode *string
ObjectKey string
StagingKey string
PipelineVersion int
Variants []Variant
CreatedAt time.Time
UpdatedAt time.Time
ReadyAt *time.Time
FailedAt *time.Time
}
// AdmissionLimits bounds accepted-but-unfinished work. Zero means unlimited.
type AdmissionLimits struct {
PerUser int
Total int
Bytes int64
}
// AttachmentDraft is a fully validated attachment awaiting persistence.
type AttachmentDraft struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
ObjectKey string
Filename string
ContentType string
SizeBytes int64
SHA256 string
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
ObjectKey string
StagingKey string
Filename string
ContentType string
SizeBytes int64
SHA256 string
Status AttachmentStatus
PipelineVersion int
}
// AttachmentState is the minimal projection used while validating message
@@ -70,6 +144,13 @@ type AttachmentKey struct {
CreatedAt time.Time
}
// StaleStaging is one staging object whose owner row is already terminal.
type StaleStaging struct {
AttachmentID uuid.UUID
StagingKey string
Status AttachmentStatus
}
// PresignRequest is the validated client intent for a direct upload.
type PresignRequest struct {
ChatID uuid.UUID
+1
View File
@@ -15,4 +15,5 @@ var (
ErrTooLarge = errors.New("payload is too large")
ErrUnsupportedMedia = errors.New("unsupported media type")
ErrIncomplete = errors.New("object is not fully uploaded")
ErrRateLimited = errors.New("too many unfinished uploads")
)
+180
View File
@@ -0,0 +1,180 @@
package domain
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/google/uuid"
)
// JobType is the fixed contract type from contracts/jobs.schema.json.
const JobType = "attachment.process.v1"
// JobStatus is the durable state of one logical processing job. Superseded is
// internal: a newer pipeline version replaced the job before it completed.
type JobStatus string
const (
JobQueued JobStatus = "queued"
JobProcessing JobStatus = "processing"
JobReady JobStatus = "ready"
JobFailed JobStatus = "failed"
JobSuperseded JobStatus = "superseded"
)
func (s JobStatus) Valid() bool {
switch s {
case JobQueued, JobProcessing, JobReady, JobFailed, JobSuperseded:
return true
default:
return false
}
}
// Job is the durable processing intent. SourceKey is the trusted object the
// worker reads: staging for a fresh upload, the original object for reprocess.
type Job struct {
ID uuid.UUID
AttachmentID uuid.UUID
PipelineVersion int
SourceKey string
Attempt int
MaxAttempts int
Status JobStatus
AvailableAt time.Time
LeaseOwner string
LeaseExpiresAt *time.Time
LastErrorCode *string
ReplayCount int
CreatedAt time.Time
UpdatedAt time.Time
CompletedAt *time.Time
}
// OutboxMessage is one pending broker publication. Payload is the exact
// jobs.schema.json document; no binary data or credentials are stored.
type OutboxMessage struct {
ID int64
JobID uuid.UUID
Attempt int
Kind string
RoutingKey string
Payload []byte
AvailableAt time.Time
PublishedAt *time.Time
ClaimedBy string
ClaimedUntil *time.Time
PublishAttempts int
LastError *string
CreatedAt time.Time
}
// JobMessage is the broker carrier defined by contracts/jobs.schema.json.
type JobMessage struct {
SchemaVersion int `json:"schema_version"`
JobID uuid.UUID `json:"job_id"`
Type string `json:"type"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
// NewJobMessage builds the only message the application is allowed to publish.
func NewJobMessage(job Job) JobMessage {
return JobMessage{
SchemaVersion: 1,
JobID: job.ID,
Type: JobType,
AttachmentID: job.AttachmentID,
PipelineVersion: job.PipelineVersion,
Attempt: job.Attempt,
CreatedAt: job.CreatedAt.UTC(),
}
}
// Validate enforces the closed schema. The worker must never trust paths or
// storage metadata from this message.
func (m JobMessage) Validate() error {
if m.SchemaVersion != 1 {
return errors.New("unsupported schema_version")
}
if m.Type != JobType {
return fmt.Errorf("unsupported type %q", m.Type)
}
if m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil {
return errors.New("job_id and attachment_id are required")
}
if m.PipelineVersion < 1 || m.Attempt < 1 {
return errors.New("pipeline_version and attempt must be >= 1")
}
if m.CreatedAt.IsZero() {
return errors.New("created_at is required")
}
return nil
}
func (m JobMessage) MarshalPayload() ([]byte, error) {
return json.Marshal(m)
}
func ParseJobMessage(payload []byte) (JobMessage, error) {
var message JobMessage
if err := json.Unmarshal(payload, &message); err != nil {
return JobMessage{}, fmt.Errorf("decode job message: %w", err)
}
if err := message.Validate(); err != nil {
return JobMessage{}, err
}
return message, nil
}
// ClaimOutcome explains why a delivery did or did not become the active
// processing attempt. The worker maps it to ack/nack without guessing.
type ClaimOutcome string
const (
ClaimAcquired ClaimOutcome = "acquired"
ClaimAlreadyReady ClaimOutcome = "already_ready"
ClaimAlreadyFailed ClaimOutcome = "already_failed"
ClaimSuperseded ClaimOutcome = "superseded"
ClaimStaleAttempt ClaimOutcome = "stale_attempt"
ClaimFutureAttempt ClaimOutcome = "future_attempt"
ClaimBusyLease ClaimOutcome = "busy_lease"
ClaimNotDue ClaimOutcome = "not_due"
ClaimVersionStale ClaimOutcome = "version_stale"
ClaimJobNotFound ClaimOutcome = "job_not_found"
ClaimSourceMissing ClaimOutcome = "source_missing"
ClaimPayloadInvalid ClaimOutcome = "payload_invalid"
)
type ClaimResult struct {
Outcome ClaimOutcome
Job Job
Attachment Attachment
}
// Job event names used by the operator audit trail.
const (
EventEnqueued = "enqueued"
EventClaimed = "claimed"
EventRetried = "retried"
EventReady = "ready"
EventFailed = "failed"
EventLeaseExpired = "lease_expired"
EventReplayed = "replayed"
EventReprocessed = "reprocessed"
EventSuperseded = "superseded"
)
+31
View File
@@ -0,0 +1,31 @@
package domain
import (
"strconv"
"github.com/google/uuid"
)
const (
StagingUploadPrefix = "staging/uploads/"
StagingVerifiedPrefix = "staging/verified/"
StagingPresignPrefix = "staging/presign/"
AttachmentsPrefix = "attachments/"
)
func StagingUploadKey(chatID, attachmentID uuid.UUID) string {
return StagingUploadPrefix + chatID.String() + "/" + attachmentID.String()
}
func StagingVerifiedKey(chatID, attachmentID uuid.UUID) string {
return StagingVerifiedPrefix + chatID.String() + "/" + attachmentID.String()
}
func OriginalObjectKey(chatID, attachmentID uuid.UUID) string {
return AttachmentsPrefix + chatID.String() + "/" + attachmentID.String() + "/original"
}
func VariantObjectKey(chatID, attachmentID uuid.UUID, pipelineVersion int, name VariantName) string {
return AttachmentsPrefix + chatID.String() + "/" + attachmentID.String() +
"/variants/v" + strconv.Itoa(pipelineVersion) + "/" + name.ObjectName()
}
+107
View File
@@ -0,0 +1,107 @@
// Package failpoint implements deterministic, environment-gated crash windows
// for the lab5 fault-injection evidence. Nothing fires unless FAULTS_ENABLED=1
// and the action matches both FAULT_<ACTION> and, for job-scoped faults,
// FAULT_TARGET_JOB. Production and CI leave the variables unset.
package failpoint
import (
"log/slog"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
)
// Enabled reports whether fault injection is explicitly switched on.
func Enabled() bool {
if os.Getenv("FAULTS_ENABLED") != "1" {
return false
}
env := os.Getenv("APP_ENV")
return env == "" || env == "dev"
}
// Check exits the process if the action is armed for this job. It must be
// called at the exact window the evidence describes; a marker line is written
// before exit so the harness can prove the fault actually fired.
func Check(action string, jobID uuid.UUID) {
if !Enabled() {
return
}
if os.Getenv("FAULT_"+action) != "exit" {
return
}
target := os.Getenv("FAULT_TARGET_JOB")
if target != "" && target != jobID.String() {
return
}
// A one-shot marker survives the container restart performed by the
// restart policy. Without it a targeted crash failpoint would loop forever.
marker := filepath.Join(os.TempDir(), "messenger-failpoint-"+action+"-"+jobID.String())
if _, err := os.Stat(marker); err == nil {
return
}
_ = os.WriteFile(marker, []byte("fired\n"), 0o600)
slog.Default().Error("failpoint fired",
slog.String("action", action),
slog.String("job_id", jobID.String()),
slog.Int("exit_code", 70),
)
_ = os.Stdout.Sync()
_ = os.Stderr.Sync()
os.Exit(70)
}
// Mode returns the armed value for an action/job pair. Unlike Check it does not
// exit; the worker uses it to inject a permanent or transient processing error.
func Mode(action string, jobID uuid.UUID) string {
if !Enabled() {
return ""
}
target := os.Getenv("FAULT_TARGET_JOB")
if target != "" && target != jobID.String() {
return ""
}
return os.Getenv("FAULT_" + action)
}
// Sleep blocks for the configured duration when the action is armed for this
// job. It is used for hang/slow-worker scenarios where the heartbeat must keep
// running (the caller controls that separately).
func Sleep(action string, jobID uuid.UUID) {
if !Enabled() {
return
}
raw := os.Getenv("FAULT_" + action)
if raw == "" {
return
}
target := os.Getenv("FAULT_TARGET_JOB")
if target != "" && target != jobID.String() {
return
}
delay, err := time.ParseDuration(raw)
if err != nil {
return
}
slog.Default().Warn("failpoint sleeping",
slog.String("action", action),
slog.String("job_id", jobID.String()),
slog.Duration("duration", delay),
)
time.Sleep(delay)
}
+314 -11
View File
@@ -1,12 +1,16 @@
// Package imagecheck validates PNG and JPEG structure without decoding the
// raster. A full decode of a 40-megapixel image would allocate hundreds of
// megabytes; attachment uploads must stay bounded. The parser verifies chunk
// CRCs and the JPEG marker/scan structure, which catches truncation and
// corruption while using constant memory.
// raster into an image. A full decode of a 40-megapixel image would allocate
// hundreds of megabytes; attachment uploads must stay bounded. For PNG the
// parser verifies chunk CRCs and streams the zlib image data to check exact
// scanline sizes and filter types without keeping the raster in memory. The
// JPEG parser verifies marker/scan structure. Both catch truncation and
// corruption while using bounded memory.
package imagecheck
import (
"bufio"
"bytes"
"compress/zlib"
"encoding/binary"
"errors"
"fmt"
@@ -73,10 +77,20 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) {
}
var (
info Info
seen = make(map[string]bool, 4)
first = true
hasIDAT bool
info Info
seen = make(map[string]bool, 4)
first = true
hasIDAT bool
paletteReady bool
trnsSeen bool
colorType byte
bitDepth byte
paletteEntries int
idat bytes.Buffer
raster []pngPass
rasterBytes int64
maxIDAT int64 = -1
lastChunk string
)
info.ContentType = "image/png"
@@ -107,11 +121,32 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) {
first = false
if seen[chunkType] && (chunkType == "IHDR" || chunkType == "PLTE" || chunkType == "IEND") {
return Info{}, corruptf("png duplicate %s chunk", chunkType)
}
if chunkType == "PLTE" && hasIDAT {
return Info{}, corruptf("png PLTE chunk after IDAT")
}
if chunkType == "IDAT" && seen["IDAT"] && lastChunk != "IDAT" {
return Info{}, corruptf("png IDAT chunks must be consecutive")
}
if err := checkCriticalChunk(chunkType, seen); err != nil {
return Info{}, err
}
data, err := readChunkData(reader, checksum, length)
var chunkSink io.Writer
if chunkType == "IDAT" {
if maxIDAT >= 0 && int64(idat.Len())+length > maxIDAT {
return Info{}, corruptf("png idat compressed data exceeds %d bytes", maxIDAT)
}
chunkSink = &idat
}
data, err := readChunkData(reader, checksum, length, chunkSink)
if err != nil {
return Info{}, corrupt("png chunk data", err)
}
@@ -133,6 +168,8 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) {
info.Width = int64(binary.BigEndian.Uint32(data[0:4]))
info.Height = int64(binary.BigEndian.Uint32(data[4:8]))
colorType = data[9]
bitDepth = data[8]
if info.Width <= 0 || info.Height <= 0 {
return Info{}, corruptf("png dimensions %dx%d", info.Width, info.Height)
@@ -143,8 +180,84 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) {
if maxPixels > 0 && (info.Width > maxPixels || info.Height > maxPixels || info.Width*info.Height > maxPixels) {
return Info{}, fmt.Errorf("%w: %dx%d", ErrTooLarge, info.Width, info.Height)
}
bound, err := pngRasterLayout(info, data[8], data[9], data[10], data[11], data[12])
if err != nil {
return Info{}, err
}
raster = bound
rasterBytes = pngRasterBytes(bound)
if rasterBytes <= 1<<40 {
// zlib overhead: ~5 bytes per 16 KiB block, 2-byte header and
// a 4-byte Adler trailer. The 1 MiB constant covers tiny
// images where the relative term vanishes.
maxIDAT = rasterBytes + rasterBytes/256 + 1<<20
}
case "IDAT":
hasIDAT = true
case "PLTE":
if colorType == 0 || colorType == 4 {
return Info{}, corruptf("png PLTE is invalid for color type %d", colorType)
}
if length == 0 || length%3 != 0 || length > 3*256 {
return Info{}, corruptf("png PLTE length %d", length)
}
if length/3 > 1<<bitDepth {
return Info{}, corruptf(
"png PLTE has %d entries for bit depth %d", length/3, bitDepth,
)
}
if trnsSeen {
return Info{}, corruptf("png PLTE chunk after tRNS")
}
paletteEntries = int(length / 3)
if !hasIDAT {
paletteReady = true
}
case "tRNS":
if trnsSeen {
return Info{}, corruptf("png duplicate tRNS chunk")
}
if hasIDAT {
return Info{}, corruptf("png tRNS chunk after IDAT")
}
switch colorType {
case 0:
if length != 2 {
return Info{}, corruptf("png tRNS length %d for color type 0", length)
}
case 2:
if length != 6 {
return Info{}, corruptf("png tRNS length %d for color type 2", length)
}
case 3:
if !paletteReady {
return Info{}, corruptf("png tRNS before PLTE")
}
if length > 256 {
return Info{}, corruptf("png tRNS length %d", length)
}
if int(length) > paletteEntries {
return Info{}, corruptf(
"png tRNS length %d exceeds %d palette entries",
length, paletteEntries,
)
}
default:
return Info{}, corruptf("png tRNS invalid for color type %d", colorType)
}
trnsSeen = true
case "IEND":
if length != 0 {
return Info{}, corruptf("png IEND length %d", length)
@@ -154,18 +267,200 @@ func validatePNG(reader *bufio.Reader, maxPixels int64) (Info, error) {
return Info{}, corruptf("png has no IDAT chunk")
}
if colorType == 3 && !paletteReady {
return Info{}, corruptf("png palette image has no PLTE before IDAT")
}
// The chunk walker proves structure, but a stream whose bytes
// merely look like a chunk layout (valid length/type/CRC) may
// still not be a PNG raster: the zlib payload must inflate to
// exactly the IHDR-declared scanlines with valid filter bytes.
if err := validateRaster(idat.Bytes(), raster); err != nil {
return Info{}, err
}
if _, err := io.Copy(io.Discard, reader); err != nil {
return Info{}, corrupt("png trailing data", err)
}
return info, nil
}
lastChunk = chunkType
}
}
// pngPass is one consecutive block of scanlines in the inflated image stream.
// Non-interlaced PNGs have a single pass; Adam7 has up to seven.
type pngPass struct {
rowBytes int64
rows int64
}
type pngPassSpec struct {
x0, y0 int64
dx, dy int64
}
var adam7Passes = []pngPassSpec{
{x0: 0, y0: 0, dx: 8, dy: 8},
{x0: 4, y0: 0, dx: 8, dy: 8},
{x0: 0, y0: 4, dx: 4, dy: 8},
{x0: 2, y0: 0, dx: 4, dy: 4},
{x0: 0, y0: 2, dx: 2, dy: 4},
{x0: 1, y0: 0, dx: 2, dy: 2},
{x0: 0, y0: 1, dx: 1, dy: 2},
}
// pngRasterLayout returns the exact inflated size and row layout for the
// IHDR. The Adam7 layout matters: a single non-interlaced formula either
// rejects valid interlaced images or accepts oversized streams.
func pngRasterLayout(
info Info,
bitDepth, colorType, compression, filter, interlace byte,
) ([]pngPass, error) {
if compression != 0 || filter != 0 {
return nil, corruptf(
"png compression/filter method %d/%d", compression, filter,
)
}
var bitsPerPixel int64
switch colorType {
case 0: // grayscale
if bitDepth != 1 && bitDepth != 2 && bitDepth != 4 && bitDepth != 8 && bitDepth != 16 {
return nil, corruptf("png invalid bit depth %d for color type 0", bitDepth)
}
bitsPerPixel = int64(bitDepth)
case 2: // truecolor
if bitDepth != 8 && bitDepth != 16 {
return nil, corruptf("png invalid bit depth %d for color type 2", bitDepth)
}
bitsPerPixel = 3 * int64(bitDepth)
case 3: // palette
if bitDepth != 1 && bitDepth != 2 && bitDepth != 4 && bitDepth != 8 {
return nil, corruptf("png invalid bit depth %d for color type 3", bitDepth)
}
bitsPerPixel = int64(bitDepth)
case 4: // grayscale + alpha
if bitDepth != 8 && bitDepth != 16 {
return nil, corruptf("png invalid bit depth %d for color type 4", bitDepth)
}
bitsPerPixel = 2 * int64(bitDepth)
case 6: // RGBA
if bitDepth != 8 && bitDepth != 16 {
return nil, corruptf("png invalid bit depth %d for color type 6", bitDepth)
}
bitsPerPixel = 4 * int64(bitDepth)
default:
return nil, corruptf("png invalid color type %d", colorType)
}
specs := []pngPassSpec{{x0: 0, y0: 0, dx: 1, dy: 1}}
switch interlace {
case 0:
case 1:
specs = adam7Passes
default:
return nil, corruptf("png invalid interlace method %d", interlace)
}
passes := make([]pngPass, 0, len(specs))
for _, spec := range specs {
if info.Width <= spec.x0 || info.Height <= spec.y0 {
continue
}
width := (info.Width - spec.x0 + spec.dx - 1) / spec.dx
rows := (info.Height - spec.y0 + spec.dy - 1) / spec.dy
rowBytes := (width*bitsPerPixel + 7) / 8
// Keep the total overflow-safe. Valid PNGs are additionally bounded
// by the pixel limit checked by the caller.
if rowBytes+1 > (1<<62)/rows {
return nil, fmt.Errorf("%w: png dimensions overflow", ErrTooLarge)
}
passes = append(passes, pngPass{rowBytes: rowBytes, rows: rows})
}
return passes, nil
}
func pngRasterBytes(passes []pngPass) int64 {
var total int64
for _, pass := range passes {
row := (pass.rowBytes + 1) * pass.rows
if total > (1<<62)-row {
return 1<<62 - 1
}
total += row
}
return total
}
// validateRaster streams the inflated image data, checking every scanline
// filter byte and requiring exactly the IHDR-declared number of rows. It
// catches zlib streams that decompress cleanly but do not describe a valid
// raster (empty payloads, bogus filter bytes) and rejects extra data as a
// decompression bomb.
func validateRaster(compressed []byte, passes []pngPass) error {
source := bytes.NewReader(compressed)
reader, err := zlib.NewReader(source)
if err != nil {
return corrupt("png idat zlib header", err)
}
for _, pass := range passes {
for range pass.rows {
var filter [1]byte
if _, err := io.ReadFull(reader, filter[:]); err != nil {
return corrupt("png scanline filter", err)
}
if filter[0] > 4 {
return corruptf("png invalid scanline filter %d", filter[0])
}
if _, err := io.CopyN(io.Discard, reader, pass.rowBytes); err != nil {
return corrupt("png scanline data", err)
}
}
}
var extra [1]byte
switch _, err := io.ReadFull(reader, extra[:]); {
case errors.Is(err, io.EOF):
// A clean zlib EOF also verifies the Adler-32 trailer. bytes.Reader
// satisfies io.ByteReader, so the inflater consumes exactly the zlib
// stream; anything left is data after the stream, which PNG forbids.
if source.Len() != 0 {
return corruptf("png idat has %d trailing bytes", source.Len())
}
return nil
case err == nil:
return fmt.Errorf("%w: png raster exceeds declared dimensions", ErrTooLarge)
default:
return corrupt("png idat zlib trailer", err)
}
}
func checkCriticalChunk(chunkType string, seen map[string]bool) error {
if chunkType == "IHDR" || chunkType == "PLTE" || chunkType == "IDAT" || chunkType == "IEND" {
seen[chunkType] = true
return nil
}
@@ -176,7 +471,7 @@ func checkCriticalChunk(chunkType string, seen map[string]bool) error {
return nil
}
func readChunkData(reader io.Reader, checksum io.Writer, length int64) ([]byte, error) {
func readChunkData(reader io.Reader, checksum io.Writer, length int64, sink io.Writer) ([]byte, error) {
if length <= 64<<10 {
data := make([]byte, length)
if _, err := io.ReadFull(reader, data); err != nil {
@@ -184,11 +479,19 @@ func readChunkData(reader io.Reader, checksum io.Writer, length int64) ([]byte,
}
_, _ = checksum.Write(data)
if sink != nil {
_, _ = sink.Write(data)
}
return data, nil
}
if _, err := io.CopyN(checksum, reader, length); err != nil {
destination := checksum
if sink != nil {
destination = io.MultiWriter(checksum, sink)
}
if _, err := io.CopyN(destination, reader, length); err != nil {
return nil, err
}
+373
View File
@@ -2,6 +2,9 @@ package imagecheck_test
import (
"bytes"
"compress/zlib"
"encoding/binary"
"hash/crc32"
"image"
"image/color"
"image/jpeg"
@@ -45,6 +48,30 @@ func encodeJPEG(t *testing.T, width, height int) []byte {
return buffer.Bytes()
}
func pngChunk(ctype string, payload []byte) []byte {
out := make([]byte, 12+len(payload))
binary.BigEndian.PutUint32(out[0:4], uint32(len(payload)))
copy(out[4:8], ctype)
copy(out[8:], payload)
checksum := crc32.NewIEEE()
_, _ = checksum.Write(out[4 : 8+len(payload)])
binary.BigEndian.PutUint32(out[8+len(payload):], checksum.Sum32())
return out
}
func pngFile(chunks ...[]byte) []byte {
data := []byte(pngSignatureForTest)
for _, chunk := range chunks {
data = append(data, chunk...)
}
return data
}
const pngSignatureForTest = "\x89PNG\r\n\x1a\n"
func TestValidatePNG(t *testing.T) {
info, err := imagecheck.Validate(bytes.NewReader(encodePNG(t, 32, 20)), 40_000_000)
require.NoError(t, err)
@@ -85,6 +112,352 @@ func TestValidateRejectsPNGCRCMismatch(t *testing.T) {
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
// Builds a structurally valid PNG (signature, well-formed IHDR/IDAT/IEND,
// correct CRCs) whose IDAT payload is not a zlib stream at all. Merely a
// stream-like structure must not make the upload pass: it has to inflate.
func TestValidateRejectsPNGIDATNotAZlibStream(t *testing.T) {
ihdr := make([]byte, 13)
binary.BigEndian.PutUint32(ihdr[0:4], 64) // width
binary.BigEndian.PutUint32(ihdr[4:8], 32) // height
ihdr[8] = 8 // bit depth
ihdr[9] = 6 // color type RGBA
data := pngFile(
pngChunk("IHDR", ihdr),
pngChunk("IDAT", []byte("this is not a zlib stream, but the CRC is right")),
pngChunk("IEND", nil),
)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func craftedPNG(t *testing.T, width, height int, interlace byte, raster []byte) []byte {
t.Helper()
return craftedPNGType(t, width, height, 6, interlace, raster)
}
func craftedPNGType(t *testing.T, width, height int, colorType, interlace byte, raster []byte) []byte {
t.Helper()
return craftedPNGDepth(t, width, height, 8, colorType, interlace, raster)
}
func craftedPNGDepth(t *testing.T, width, height int, bitDepth, colorType, interlace byte, raster []byte) []byte {
t.Helper()
return pngFile(
pngChunk("IHDR", pngIHDR(width, height, bitDepth, colorType, interlace)),
pngChunk("IDAT", zlibBytes(t, raster)),
pngChunk("IEND", nil),
)
}
func pngIHDR(width, height int, bitDepth, colorType, interlace byte) []byte {
ihdr := make([]byte, 13)
binary.BigEndian.PutUint32(ihdr[0:4], uint32(width))
binary.BigEndian.PutUint32(ihdr[4:8], uint32(height))
ihdr[8] = bitDepth
ihdr[9] = colorType
ihdr[12] = interlace
return ihdr
}
func zlibBytes(t *testing.T, raster []byte) []byte {
t.Helper()
var compressed bytes.Buffer
writer := zlib.NewWriter(&compressed)
_, err := writer.Write(raster)
require.NoError(t, err)
require.NoError(t, writer.Close())
return compressed.Bytes()
}
func TestValidateRejectsPNGEmptyRaster(t *testing.T) {
data := craftedPNG(t, 2, 2, 0, nil)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsPNGInvalidScanlineFilter(t *testing.T) {
raster := []byte{5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
data := craftedPNG(t, 2, 2, 0, raster)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsPNGPaletteWithoutPLTE(t *testing.T) {
raster := []byte{0, 0, 0, 0, 0, 0} // two rows of two 8-bit palette indexes
data := craftedPNGType(t, 2, 2, 3, 0, raster)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateAcceptsPalettePNG(t *testing.T) {
palette := color.Palette{color.RGBA{R: 255, A: 255}, color.RGBA{G: 255, A: 255}, color.RGBA{B: 255, A: 255}}
source := image.NewPaletted(image.Rect(0, 0, 13, 5), palette)
for x := range 13 {
for y := range 5 {
source.SetColorIndex(x, y, uint8((x+y)%len(palette)))
}
}
var buffer bytes.Buffer
require.NoError(t, png.Encode(&buffer, source))
info, err := imagecheck.Validate(bytes.NewReader(buffer.Bytes()), 40_000_000)
require.NoError(t, err)
require.Equal(t, int64(13), info.Width)
require.Equal(t, int64(5), info.Height)
}
func TestValidateRejectsPNGInvalidColorDepthCombination(t *testing.T) {
data := craftedPNGDepth(t, 1, 1, 1, 6, 0, []byte{0, 0, 0})
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsPNGDuplicateIHDR(t *testing.T) {
ihdr := pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0))
data := pngFile(
ihdr,
ihdr,
pngChunk("IDAT", zlibBytes(t, []byte{0, 0, 0, 0, 0})),
pngChunk("IEND", nil),
)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsPNGDuplicatePLTE(t *testing.T) {
plte := pngChunk("PLTE", []byte{255, 0, 0})
data := pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 3, 0)),
plte,
plte,
pngChunk("IDAT", zlibBytes(t, []byte{0, 0})),
pngChunk("IEND", nil),
)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsPNGNonConsecutiveIDAT(t *testing.T) {
compressed := zlibBytes(t, []byte{0, 0, 0, 0, 0})
middle := len(compressed) / 2
data := pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)),
pngChunk("IDAT", compressed[:middle]),
pngChunk("tEXt", []byte("k\x00v")),
pngChunk("IDAT", compressed[middle:]),
pngChunk("IEND", nil),
)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsPNGTrailingIDATData(t *testing.T) {
idat := append(zlibBytes(t, []byte{0, 0, 0, 0, 0}), []byte("junk")...)
data := pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)),
pngChunk("IDAT", idat),
pngChunk("IEND", nil),
)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateAcceptsSplitIDAT(t *testing.T) {
compressed := zlibBytes(t, []byte{0, 0, 0, 0, 0})
middle := len(compressed) / 2
data := pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)),
pngChunk("IDAT", compressed[:middle]),
pngChunk("IDAT", compressed[middle:]),
pngChunk("IEND", nil),
)
info, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.NoError(t, err)
require.Equal(t, int64(1), info.Width)
}
func TestValidateAcceptsStandardEncodings(t *testing.T) {
bounds := image.Rect(0, 0, 7, 5)
images := []image.Image{
image.NewGray(bounds),
image.NewGray16(bounds),
image.NewRGBA(bounds),
image.NewNRGBA(bounds),
image.NewNRGBA64(bounds),
}
for _, source := range images {
var buffer bytes.Buffer
require.NoError(t, png.Encode(&buffer, source))
info, err := imagecheck.Validate(bytes.NewReader(buffer.Bytes()), 40_000_000)
require.NoError(t, err)
require.Equal(t, int64(7), info.Width)
require.Equal(t, int64(5), info.Height)
}
}
func TestValidateRejectsPNGPLTEEntriesExceedBitDepth(t *testing.T) {
data := pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 1, 3, 0)),
pngChunk("PLTE", []byte{255, 0, 0, 0, 255, 0, 0, 0, 255}),
pngChunk("IDAT", zlibBytes(t, []byte{0, 0})),
pngChunk("IEND", nil),
)
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt)
}
func TestValidateRejectsInvalidTRNS(t *testing.T) {
rgb := pngChunk("IDAT", zlibBytes(t, []byte{0, 10, 20, 30}))
gray := pngChunk("IDAT", zlibBytes(t, []byte{0, 10}))
rgba := pngChunk("IDAT", zlibBytes(t, []byte{0, 10, 20, 30, 255}))
paletted := pngChunk("IDAT", zlibBytes(t, []byte{0, 0}))
cases := map[string][]byte{
"tRNS on RGBA": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 6, 0)),
pngChunk("tRNS", []byte{0}),
rgba,
pngChunk("IEND", nil),
),
"RGB length": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)),
pngChunk("tRNS", []byte{0}),
rgb,
pngChunk("IEND", nil),
),
"gray length": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 0, 0)),
pngChunk("tRNS", []byte{0}),
gray,
pngChunk("IEND", nil),
),
"palette before PLTE": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 3, 0)),
pngChunk("tRNS", []byte{0}),
pngChunk("PLTE", []byte{255, 0, 0, 0, 255, 0}),
paletted,
pngChunk("IEND", nil),
),
"palette tRNS longer than PLTE": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 3, 0)),
pngChunk("PLTE", []byte{255, 0, 0}),
pngChunk("tRNS", []byte{0, 0}),
paletted,
pngChunk("IEND", nil),
),
"duplicate tRNS": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)),
pngChunk("tRNS", make([]byte, 6)),
pngChunk("tRNS", make([]byte, 6)),
rgb,
pngChunk("IEND", nil),
),
"tRNS after IDAT": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)),
rgb,
pngChunk("tRNS", make([]byte, 6)),
pngChunk("IEND", nil),
),
}
for name, data := range cases {
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.ErrorIs(t, err, imagecheck.ErrCorrupt, name)
}
}
func TestValidateAcceptsValidTRNS(t *testing.T) {
cases := map[string][]byte{
"RGB": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 2, 0)),
pngChunk("tRNS", make([]byte, 6)),
pngChunk("IDAT", zlibBytes(t, []byte{0, 10, 20, 30})),
pngChunk("IEND", nil),
),
"gray": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 8, 0, 0)),
pngChunk("tRNS", make([]byte, 2)),
pngChunk("IDAT", zlibBytes(t, []byte{0, 10})),
pngChunk("IEND", nil),
),
"palette": pngFile(
pngChunk("IHDR", pngIHDR(1, 1, 1, 3, 0)),
pngChunk("PLTE", []byte{255, 0, 0, 0, 255, 0}),
pngChunk("tRNS", []byte{128}),
pngChunk("IDAT", zlibBytes(t, []byte{0, 0})),
pngChunk("IEND", nil),
),
}
for name, data := range cases {
_, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.NoError(t, err, name)
}
}
// A valid Adam7 stream for a tall, narrow image exceeds the old
// (rowBytes+1)*(height+8) guess; the exact pass layout must accept it.
func TestValidateAcceptsAdam7PNG(t *testing.T) {
const width, height = 2, 256
var raster bytes.Buffer
passes := [][4]int{
{0, 0, 8, 8},
{4, 0, 8, 8},
{0, 4, 4, 8},
{2, 0, 4, 4},
{0, 2, 2, 4},
{1, 0, 2, 2},
{0, 1, 1, 2},
}
for _, pass := range passes {
x0, y0, dx, dy := pass[0], pass[1], pass[2], pass[3]
if width <= x0 || height <= y0 {
continue
}
for y := y0; y < height; y += dy {
raster.WriteByte(0) // filter: none
for x := x0; x < width; x += dx {
raster.Write([]byte{byte(x), byte(y), 128, 255})
}
}
}
data := craftedPNG(t, width, height, 1, raster.Bytes())
info, err := imagecheck.Validate(bytes.NewReader(data), 40_000_000)
require.NoError(t, err)
require.Equal(t, int64(width), info.Width)
require.Equal(t, int64(height), info.Height)
}
func TestValidateRejectsTruncatedJPEG(t *testing.T) {
data := encodeJPEG(t, 64, 64)
_, err := imagecheck.Validate(bytes.NewReader(data[:len(data)-32]), 40_000_000)
+118
View File
@@ -0,0 +1,118 @@
package imageproc
import (
"encoding/binary"
"io"
)
const (
exifMaxBytes = 64 << 10
exifTagOrient = 0x0112
)
// readEXIFOrientation extracts only IFD0 Orientation from a JPEG APP1 segment.
// The parser is deliberately small and bounded: no TIFF offsets are trusted and
// the whole segment is capped at 64 KiB, so attacker-controlled EXIF cannot make
// the worker allocate unbounded memory.
func readEXIFOrientation(r io.Reader) int {
data, err := io.ReadAll(io.LimitReader(r, exifMaxBytes))
if err != nil || len(data) < 4 || data[0] != 0xff || data[1] != 0xd8 {
return 1
}
pos := 2
for pos+4 <= len(data) {
if data[pos] != 0xff {
return 1
}
marker := data[pos+1]
pos += 2
// Standalone markers without a length field.
if marker == 0xd8 || marker == 0xd9 || marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7) {
continue
}
if pos+2 > len(data) {
return 1
}
length := int(binary.BigEndian.Uint16(data[pos : pos+2]))
if length < 2 || pos+length > len(data) {
return 1
}
segment := data[pos+2 : pos+length]
pos += length
if marker != 0xe1 || len(segment) < 6 {
continue
}
if string(segment[:6]) != "Exif\x00\x00" {
continue
}
return parseTIFFOrientation(segment[6:])
}
return 1
}
func parseTIFFOrientation(tiff []byte) int {
if len(tiff) < 8 {
return 1
}
var order binary.ByteOrder
switch string(tiff[:2]) {
case "II":
order = binary.LittleEndian
case "MM":
order = binary.BigEndian
default:
return 1
}
if order.Uint16(tiff[2:4]) != 42 {
return 1
}
ifdOffset := int(order.Uint32(tiff[4:8]))
if ifdOffset < 8 || ifdOffset+2 > len(tiff) {
return 1
}
count := int(order.Uint16(tiff[ifdOffset : ifdOffset+2]))
if count > 512 {
count = 512
}
entry := ifdOffset + 2
for range count {
if entry+12 > len(tiff) {
return 1
}
tag := order.Uint16(tiff[entry : entry+2])
typ := order.Uint16(tiff[entry+2 : entry+4])
n := order.Uint32(tiff[entry+4 : entry+8])
if tag == exifTagOrient && typ == 3 && n == 1 {
value := order.Uint16(tiff[entry+8 : entry+10])
if value >= 1 && value <= 8 {
return int(value)
}
return 1
}
entry += 12
}
return 1
}
+259
View File
@@ -0,0 +1,259 @@
// Package imageproc turns one decoded original into the contract variants.
// Output bytes are deterministic for a given pipeline version, source bytes and
// encoder versions; the worker uses that property for idempotent retries.
package imageproc
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"image"
"image/color"
"image/jpeg"
_ "image/png" // register the PNG decoder
"io"
"math"
"os"
"github.com/gen2brain/webp"
xdraw "golang.org/x/image/draw"
"mtusi-messenger/internal/domain"
)
const (
thumbnailMax = 256
cropSize = 128
optimizedMax = 1280
defaultMethod = 2
)
// Params are immutable per pipeline version. They are compiled into the binary
// so a retry can never silently produce different bytes than the first attempt.
type Params struct {
JPEGQuality int
WebPQuality int
WebPMethod int
}
// Pipeline returns the versioned encoder parameters. Only v1 and v2 are
// defined; the course baseline is v1.
func Pipeline(version int) (Params, bool) {
switch version {
case 1:
return Params{JPEGQuality: 85, WebPQuality: 80, WebPMethod: defaultMethod}, true
case 2:
return Params{JPEGQuality: 70, WebPQuality: 65, WebPMethod: 4}, true
default:
return Params{}, false
}
}
// Image is one encoded variant.
type Image struct {
Name domain.VariantName
ContentType string
Data []byte
SizeBytes int64
Width int
Height int
SHA256 string
}
// Load decodes a PNG/JPEG file, applies EXIF orientation and composites alpha
// on white. The returned image is always *image.RGBA, which bounds the variant
// encoders to 8-bit output.
func Load(path string, maxPixels int64) (image.Image, string, error) {
file, err := os.Open(path)
if err != nil {
return nil, "", fmt.Errorf("open source: %w", err)
}
defer file.Close()
cfg, format, err := image.DecodeConfig(file)
if err != nil {
return nil, "", fmt.Errorf("decode config: %w", err)
}
if cfg.Width <= 0 || cfg.Height <= 0 {
return nil, "", fmt.Errorf("invalid dimensions %dx%d", cfg.Width, cfg.Height)
}
if maxPixels > 0 && int64(cfg.Width)*int64(cfg.Height) > maxPixels {
return nil, "", fmt.Errorf("image has %d pixels, limit %d", int64(cfg.Width)*int64(cfg.Height), maxPixels)
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return nil, "", fmt.Errorf("rewind source: %w", err)
}
decoded, _, err := image.Decode(file)
if err != nil {
return nil, "", fmt.Errorf("decode image: %w", err)
}
orientation := 1
if format == "jpeg" {
if _, err := file.Seek(0, io.SeekStart); err == nil {
orientation = readEXIFOrientation(file)
}
}
return toWhiteRGBA(decoded, orientation), format, nil
}
func toWhiteRGBA(src image.Image, orientation int) image.Image {
transformed := orient(src, orientation)
bounds := transformed.Bounds()
dst := image.NewRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
xdraw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, xdraw.Src)
xdraw.Draw(dst, dst.Bounds(), transformed, bounds.Min, xdraw.Over)
return dst
}
// orient applies the eight EXIF orientation values. The result has the same
// bounds as the source except for 90-degree rotations.
func orient(src image.Image, orientation int) image.Image {
if orientation < 2 || orientation > 8 {
return src
}
b := src.Bounds()
w, h := b.Dx(), b.Dy()
width, height := w, h
if orientation >= 5 {
width, height = h, w
}
dst := image.NewRGBA(image.Rect(0, 0, width, height))
for y := range h {
for x := range w {
nx, ny := orientPoint(orientation, x, y, w, h)
dst.Set(nx, ny, src.At(b.Min.X+x, b.Min.Y+y))
}
}
return dst
}
func orientPoint(orientation, x, y, w, h int) (int, int) {
switch orientation {
case 2:
return w - 1 - x, y
case 3:
return w - 1 - x, h - 1 - y
case 4:
return x, h - 1 - y
case 5:
return y, x
case 6:
return h - 1 - y, x
case 7:
return h - 1 - y, w - 1 - x
case 8:
return y, w - 1 - x
default:
return x, y
}
}
// Thumbnail fits inside 256x256 without upscaling; dimensions are floored and
// never below 1 px.
func Thumbnail(src image.Image, params Params) (Image, error) {
return encodeJPEG(fit(src, thumbnailMax, thumbnailMax, false), domain.VariantThumbnail, params)
}
// Crop takes the centered square and scales it to 128x128. Upscaling small
// sources is allowed by the contract.
func Crop(src image.Image, params Params) (Image, error) {
b := src.Bounds()
side := b.Dx()
if b.Dy() < side {
side = b.Dy()
}
left := b.Min.X + (b.Dx()-side)/2
top := b.Min.Y + (b.Dy()-side)/2
square := image.NewRGBA(image.Rect(0, 0, side, side))
xdraw.Draw(square, square.Bounds(), src, image.Pt(left, top), xdraw.Src)
dst := image.NewRGBA(image.Rect(0, 0, cropSize, cropSize))
xdraw.CatmullRom.Scale(dst, dst.Bounds(), square, square.Bounds(), xdraw.Over, nil)
return encodeJPEG(dst, domain.VariantCrop, params)
}
// Optimized fits inside 1280x1280 without upscaling and encodes WebP. Metadata
// is stripped because the output is a fresh re-encode.
func Optimized(src image.Image, params Params) (Image, error) {
dst := fit(src, optimizedMax, optimizedMax, false)
var buf bytes.Buffer
if err := webp.Encode(&buf, dst, webp.Options{
Quality: params.WebPQuality,
Method: params.WebPMethod,
}); err != nil {
return Image{}, fmt.Errorf("encode webp: %w", err)
}
return newImage(domain.VariantOptimized, "image/webp", buf.Bytes(), dst.Bounds().Dx(), dst.Bounds().Dy()), nil
}
func encodeJPEG(dst image.Image, name domain.VariantName, params Params) (Image, error) {
var buf bytes.Buffer
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: params.JPEGQuality}); err != nil {
return Image{}, fmt.Errorf("encode jpeg: %w", err)
}
return newImage(name, "image/jpeg", buf.Bytes(), dst.Bounds().Dx(), dst.Bounds().Dy()), nil
}
func newImage(name domain.VariantName, contentType string, data []byte, width, height int) Image {
sum := sha256.Sum256(data)
return Image{
Name: name,
ContentType: contentType,
Data: data,
SizeBytes: int64(len(data)),
Width: width,
Height: height,
SHA256: hex.EncodeToString(sum[:]),
}
}
// fit scales src into maxWidth x maxHeight preserving aspect ratio, floors the
// dimensions, enforces at least 1 px and, unless allowUpscale is set, never
// enlarges the source.
func fit(src image.Image, maxWidth, maxHeight int, allowUpscale bool) *image.RGBA {
b := src.Bounds()
w, h := b.Dx(), b.Dy()
scale := math.Min(float64(maxWidth)/float64(w), float64(maxHeight)/float64(h))
if !allowUpscale && scale > 1 {
scale = 1
}
width := int(math.Floor(float64(w) * scale))
height := int(math.Floor(float64(h) * scale))
if width < 1 {
width = 1
}
if height < 1 {
height = 1
}
dst := image.NewRGBA(image.Rect(0, 0, width, height))
xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, b, xdraw.Over, nil)
return dst
}
+222
View File
@@ -0,0 +1,222 @@
package imageproc_test
import (
"bytes"
"encoding/binary"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"path/filepath"
"testing"
"github.com/gen2brain/webp"
"github.com/stretchr/testify/require"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/imageproc"
)
func writePNG(t *testing.T, width, height int, alpha bool) string {
t.Helper()
img := image.NewNRGBA(image.Rect(0, 0, width, height))
for x := range width {
for y := range height {
a := uint8(255)
if alpha && x < width/2 && y < height/2 {
a = 0
}
img.Set(x, y, color.NRGBA{R: 200, G: 40, B: 20, A: a})
}
}
var buf bytes.Buffer
require.NoError(t, png.Encode(&buf, img))
path := filepath.Join(t.TempDir(), "source.png")
require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o600))
return path
}
func writeJPEG(t *testing.T, width, height, orientation int) string {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
for x := range width {
for y := range height {
img.Set(x, y, color.RGBA{R: 200, G: 40, B: 20, A: 255})
}
}
var raw bytes.Buffer
require.NoError(t, jpeg.Encode(&raw, img, &jpeg.Options{Quality: 95}))
data := append([]byte(nil), raw.Bytes()...)
if orientation > 1 {
data = insertEXIFOrientation(t, data, uint16(orientation))
}
path := filepath.Join(t.TempDir(), "source.jpg")
require.NoError(t, os.WriteFile(path, data, 0o600))
return path
}
func insertEXIFOrientation(t *testing.T, jpegBytes []byte, orientation uint16) []byte {
t.Helper()
// TIFF: II, 42, IFD0 at offset 8, one entry (0x0112 SHORT 1 = orientation).
tiff := make([]byte, 8+2+12+4)
binary.LittleEndian.PutUint16(tiff[0:2], 0x4949)
binary.LittleEndian.PutUint16(tiff[2:4], 42)
binary.LittleEndian.PutUint32(tiff[4:8], 8)
binary.LittleEndian.PutUint16(tiff[8:10], 1)
binary.LittleEndian.PutUint16(tiff[10:12], 0x0112)
binary.LittleEndian.PutUint16(tiff[12:14], 3)
binary.LittleEndian.PutUint32(tiff[14:18], 1)
binary.LittleEndian.PutUint16(tiff[18:20], orientation)
payload := append([]byte("Exif\x00\x00"), tiff...)
segment := make([]byte, 4+len(payload))
segment[0] = 0xff
segment[1] = 0xe1
binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2))
copy(segment[4:], payload)
// Insert immediately after SOI (before the first DQT/SOF).
out := append([]byte(nil), jpegBytes[:2]...)
out = append(out, segment...)
out = append(out, jpegBytes[2:]...)
return out
}
func decodeConfig(t *testing.T, data []byte) image.Config {
t.Helper()
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
require.NoError(t, err)
return cfg
}
func TestThumbnailContract(t *testing.T) {
params, ok := imageproc.Pipeline(1)
require.True(t, ok)
cases := []struct {
name string
width int
height int
expected [2]int
}{
{"320x200", 320, 200, [2]int{256, 160}},
{"no upscale 64x64", 64, 64, [2]int{64, 64}},
{"no upscale 64x40", 64, 40, [2]int{64, 40}},
{"floor 1000x10", 1000, 10, [2]int{256, 2}},
{"height limited", 100, 320, [2]int{80, 256}},
{"minimum 1x1", 1, 1, [2]int{1, 1}},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
img, _, err := imageproc.Load(writePNG(t, test.width, test.height, false), 40_000_000)
require.NoError(t, err)
thumb, err := imageproc.Thumbnail(img, params)
require.NoError(t, err)
cfg := decodeConfig(t, thumb.Data)
require.Equal(t, test.expected[0], cfg.Width)
require.Equal(t, test.expected[1], cfg.Height)
require.Equal(t, test.expected[0], thumb.Width)
require.Equal(t, test.expected[1], thumb.Height)
require.Equal(t, "image/jpeg", thumb.ContentType)
require.NotContains(t, string(thumb.Data), "Exif")
})
}
}
func TestCropAndOptimized(t *testing.T) {
params, ok := imageproc.Pipeline(1)
require.True(t, ok)
img, _, err := imageproc.Load(writePNG(t, 320, 200, false), 40_000_000)
require.NoError(t, err)
crop, err := imageproc.Crop(img, params)
require.NoError(t, err)
cfg := decodeConfig(t, crop.Data)
require.Equal(t, 128, cfg.Width)
require.Equal(t, 128, cfg.Height)
optimized, err := imageproc.Optimized(img, params)
require.NoError(t, err)
cfg = decodeConfig(t, optimized.Data)
require.Equal(t, 320, cfg.Width)
require.Equal(t, 200, cfg.Height)
require.Equal(t, "image/webp", optimized.ContentType)
decoded, err := webp.Decode(bytes.NewReader(optimized.Data))
require.NoError(t, err)
require.Equal(t, 320, decoded.Bounds().Dx())
require.Equal(t, 200, decoded.Bounds().Dy())
}
func TestEXIFOrientationChangesThumbnailDimensions(t *testing.T) {
params, ok := imageproc.Pipeline(1)
require.True(t, ok)
for _, orientation := range []int{6, 8} {
img, _, err := imageproc.Load(writeJPEG(t, 320, 200, orientation), 40_000_000)
require.NoError(t, err)
thumb, err := imageproc.Thumbnail(img, params)
require.NoError(t, err)
cfg := decodeConfig(t, thumb.Data)
require.Equal(t, 160, cfg.Width, "orientation %d", orientation)
require.Equal(t, 256, cfg.Height, "orientation %d", orientation)
}
}
func TestAlphaCompositedOnWhite(t *testing.T) {
params, ok := imageproc.Pipeline(1)
require.True(t, ok)
img, _, err := imageproc.Load(writePNG(t, 100, 100, true), 40_000_000)
require.NoError(t, err)
thumb, err := imageproc.Thumbnail(img, params)
require.NoError(t, err)
decoded, err := jpeg.Decode(bytes.NewReader(thumb.Data))
require.NoError(t, err)
r, g, b, _ := decoded.At(0, 0).RGBA()
require.Greater(t, r, uint32(0xf000), "transparent corner must be white")
require.Greater(t, g, uint32(0xf000), "transparent corner must be white")
require.Greater(t, b, uint32(0xf000), "transparent corner must be white")
}
func TestPipelineVersionsDiffer(t *testing.T) {
v1, ok := imageproc.Pipeline(1)
require.True(t, ok)
v2, ok := imageproc.Pipeline(2)
require.True(t, ok)
require.NotEqual(t, v1, v2)
_, ok = imageproc.Pipeline(3)
require.False(t, ok)
}
func TestVariantNames(t *testing.T) {
require.Equal(t, "image/jpeg", domain.VariantThumbnail.ContentType())
require.Equal(t, "image/jpeg", domain.VariantCrop.ContentType())
require.Equal(t, "image/webp", domain.VariantOptimized.ContentType())
require.Equal(t, "thumbnail.jpg", domain.VariantThumbnail.ObjectName())
require.Equal(t, "optimized.webp", domain.VariantOptimized.ObjectName())
}
+168
View File
@@ -0,0 +1,168 @@
// Package outbox publishes the durable job intents with publisher confirms and
// reclaims abandoned relay claims and expired worker leases.
package outbox
import (
"context"
"log/slog"
"time"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/failpoint"
)
type Repository interface {
ClaimOutbox(ctx context.Context, owner string, claimTTL time.Duration, limit int) ([]domain.OutboxMessage, error)
MarkPublished(ctx context.Context, ids []int64) error
ReleaseOutbox(ctx context.Context, id int64, reason string) error
ReapOutboxClaims(ctx context.Context) error
ReapExpiredLeases(ctx context.Context, limit int) (int, error)
ReconcileQueuedJobs(ctx context.Context, olderThan time.Duration, limit int) (int, error)
PruneOutbox(ctx context.Context, retention time.Duration) error
OldestUnpublished(ctx context.Context) (time.Time, error)
}
type Publisher interface {
PublishConfirmed(ctx context.Context, routingKey string, message domain.JobMessage) error
}
type Config struct {
Owner string
PollInterval time.Duration
ClaimTTL time.Duration
Batch int
ReaperInterval time.Duration
ReconcileAfter time.Duration
Retention time.Duration
}
type Relay struct {
repo Repository
publisher Publisher
config Config
log *slog.Logger
lastPublished time.Time
lastReap time.Time
}
func New(repo Repository, publisher Publisher, config Config, log *slog.Logger) *Relay {
if log == nil {
log = slog.Default()
}
return &Relay{repo: repo, publisher: publisher, config: config, log: log}
}
func (r *Relay) Run(ctx context.Context) {
poll := time.NewTicker(r.config.PollInterval)
defer poll.Stop()
reaper := time.NewTicker(r.config.ReaperInterval)
defer reaper.Stop()
for {
select {
case <-ctx.Done():
return
case <-poll.C:
if _, err := r.RunOnce(ctx); err != nil {
r.log.WarnContext(ctx, "outbox publish cycle failed", slog.Any("error", err))
}
case <-reaper.C:
r.reap(ctx)
}
}
}
// RunOnce claims due rows, publishes each with a broker confirm and marks only
// confirmed rows. A crash between confirm and mark causes a duplicate publish,
// which the worker tolerates by CAS.
func (r *Relay) RunOnce(ctx context.Context) (int, error) {
messages, err := r.repo.ClaimOutbox(ctx, r.config.Owner, r.config.ClaimTTL, r.config.Batch)
if err != nil {
return 0, err
}
published := make([]int64, 0, len(messages))
for _, message := range messages {
job, err := domain.ParseJobMessage(message.Payload)
if err != nil {
// A malformed payload is an application bug, not a transient
// failure. Keep the row claim until it expires and log loudly; the
// operator can inspect the outbox.
r.log.ErrorContext(ctx, "outbox payload is invalid",
slog.Int64("outbox_id", message.ID),
slog.Any("error", err),
)
_ = r.repo.ReleaseOutbox(ctx, message.ID, "invalid payload")
continue
}
failpoint.Check("OUTBOX_AFTER_COMMIT", job.JobID)
if err := r.publisher.PublishConfirmed(ctx, message.RoutingKey, job); err != nil {
_ = r.repo.ReleaseOutbox(ctx, message.ID, err.Error())
continue
}
published = append(published, message.ID)
r.lastPublished = time.Now()
}
if err := r.repo.MarkPublished(ctx, published); err != nil {
return len(published), err
}
return len(published), nil
}
func (r *Relay) reap(ctx context.Context) {
if err := r.repo.ReapOutboxClaims(ctx); err != nil {
r.log.WarnContext(ctx, "outbox claim reaper failed", slog.Any("error", err))
}
reaped, err := r.repo.ReapExpiredLeases(ctx, 100)
if err != nil {
r.log.WarnContext(ctx, "lease reaper failed", slog.Any("error", err))
} else if reaped > 0 {
r.log.InfoContext(ctx, "expired job leases requeued", slog.Int("count", reaped))
}
// The broker is not the source of truth. If a delivery was dead-lettered
// or lost, the queued job has no unpublished outbox row; republish it.
if r.config.ReconcileAfter > 0 {
reconciled, err := r.repo.ReconcileQueuedJobs(ctx, r.config.ReconcileAfter, 100)
if err != nil {
r.log.WarnContext(ctx, "queued-job reconciliation failed", slog.Any("error", err))
} else if reconciled > 0 {
r.log.InfoContext(ctx, "lost queued jobs reconciled", slog.Int("count", reconciled))
}
}
if r.config.Retention > 0 {
if err := r.repo.PruneOutbox(ctx, r.config.Retention); err != nil {
r.log.WarnContext(ctx, "outbox retention failed", slog.Any("error", err))
}
}
r.lastReap = time.Now()
}
// Health returns the age of the oldest unpublished intent. A bounded age is a
// better relay readiness signal than "connected to broker".
func (r *Relay) Health(ctx context.Context) (oldest time.Duration, err error) {
oldestAt, err := r.repo.OldestUnpublished(ctx)
if err != nil {
return 0, err
}
if oldestAt.IsZero() {
return 0, nil
}
return time.Since(oldestAt), nil
}
@@ -22,6 +22,8 @@ type Object struct {
// because AWS SigV4 may need to rewind the payload; the proxy upload path always
// passes an *os.File.
type Store interface {
// Ping verifies that the bucket is reachable and the credentials are valid.
Ping(ctx context.Context) error
Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string, metadata map[string]string) error
Open(ctx context.Context, key string) (io.ReadCloser, Object, error)
Head(ctx context.Context, key string) (Object, error)
+13
View File
@@ -91,6 +91,19 @@ func New(cfg Config) (*S3, error) {
return st, nil
}
func (s *S3) Ping(ctx context.Context) error {
// HeadBucket needs bucket-level permissions that the application identity
// intentionally does not have. A HeadObject on a sentinel staging key is
// enough to prove connectivity and the object-level credentials; a missing
// sentinel is the expected healthy result.
_, err := s.Head(ctx, "staging/.health/ping")
if err == nil || errors.Is(err, domain.ErrNotFound) {
return nil
}
return err
}
func (s *S3) Put(
ctx context.Context,
key string,
+393 -272
View File
@@ -7,7 +7,6 @@ package mocks
import (
"context"
"mtusi-messenger/internal/domain"
"time"
"github.com/google/uuid"
mock "github.com/stretchr/testify/mock"
@@ -49,53 +48,65 @@ func (_m *MockRepository) EXPECT() *MockRepository_Expecter {
return &MockRepository_Expecter{mock: &_m.Mock}
}
// Create provides a mock function for the type MockRepository
func (_mock *MockRepository) Create(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error) {
ret := _mock.Called(ctx, draft)
// Admission provides a mock function for the type MockRepository
func (_mock *MockRepository) Admission(ctx context.Context, ownerID uuid.UUID) (int, int, int64, error) {
ret := _mock.Called(ctx, ownerID)
if len(ret) == 0 {
panic("no return value specified for Create")
panic("no return value specified for Admission")
}
var r0 domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft) (domain.Attachment, error)); ok {
return returnFunc(ctx, draft)
var r0 int
var r1 int
var r2 int64
var r3 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID) (int, int, int64, error)); ok {
return returnFunc(ctx, ownerID)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft) domain.Attachment); ok {
r0 = returnFunc(ctx, draft)
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID) int); ok {
r0 = returnFunc(ctx, ownerID)
} else {
r0 = ret.Get(0).(domain.Attachment)
r0 = ret.Get(0).(int)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, domain.AttachmentDraft) error); ok {
r1 = returnFunc(ctx, draft)
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID) int); ok {
r1 = returnFunc(ctx, ownerID)
} else {
r1 = ret.Error(1)
r1 = ret.Get(1).(int)
}
return r0, r1
if returnFunc, ok := ret.Get(2).(func(context.Context, uuid.UUID) int64); ok {
r2 = returnFunc(ctx, ownerID)
} else {
r2 = ret.Get(2).(int64)
}
if returnFunc, ok := ret.Get(3).(func(context.Context, uuid.UUID) error); ok {
r3 = returnFunc(ctx, ownerID)
} else {
r3 = ret.Error(3)
}
return r0, r1, r2, r3
}
// MockRepository_Create_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Create'
type MockRepository_Create_Call struct {
// MockRepository_Admission_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Admission'
type MockRepository_Admission_Call struct {
*mock.Call
}
// Create is a helper method to define mock.On call
// Admission is a helper method to define mock.On call
// - ctx context.Context
// - draft domain.AttachmentDraft
func (_e *MockRepository_Expecter) Create(ctx any, draft any) *MockRepository_Create_Call {
return &MockRepository_Create_Call{Call: _e.mock.On("Create", ctx, draft)}
// - ownerID uuid.UUID
func (_e *MockRepository_Expecter) Admission(ctx any, ownerID any) *MockRepository_Admission_Call {
return &MockRepository_Admission_Call{Call: _e.mock.On("Admission", ctx, ownerID)}
}
func (_c *MockRepository_Create_Call) Run(run func(ctx context.Context, draft domain.AttachmentDraft)) *MockRepository_Create_Call {
func (_c *MockRepository_Admission_Call) Run(run func(ctx context.Context, ownerID uuid.UUID)) *MockRepository_Admission_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 domain.AttachmentDraft
var arg1 uuid.UUID
if args[1] != nil {
arg1 = args[1].(domain.AttachmentDraft)
arg1 = args[1].(uuid.UUID)
}
run(
arg0,
@@ -105,12 +116,264 @@ func (_c *MockRepository_Create_Call) Run(run func(ctx context.Context, draft do
return _c
}
func (_c *MockRepository_Create_Call) Return(attachment domain.Attachment, err error) *MockRepository_Create_Call {
func (_c *MockRepository_Admission_Call) Return(userPending int, totalPending int, stagedBytes int64, err error) *MockRepository_Admission_Call {
_c.Call.Return(userPending, totalPending, stagedBytes, err)
return _c
}
func (_c *MockRepository_Admission_Call) RunAndReturn(run func(ctx context.Context, ownerID uuid.UUID) (int, int, int64, error)) *MockRepository_Admission_Call {
_c.Call.Return(run)
return _c
}
// CompletePresign provides a mock function for the type MockRepository
func (_mock *MockRepository) CompletePresign(ctx context.Context, ownerID uuid.UUID, attachmentID uuid.UUID, verifiedKey string, finalKey string, pipelineVersion int, maxAttempts int, jobID uuid.UUID) (domain.Attachment, error) {
ret := _mock.Called(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID)
if len(ret) == 0 {
panic("no return value specified for CompletePresign")
}
var r0 domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, string, string, int, int, uuid.UUID) (domain.Attachment, error)); ok {
return returnFunc(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, string, string, int, int, uuid.UUID) domain.Attachment); ok {
r0 = returnFunc(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID)
} else {
r0 = ret.Get(0).(domain.Attachment)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, uuid.UUID, string, string, int, int, uuid.UUID) error); ok {
r1 = returnFunc(ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_CompletePresign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CompletePresign'
type MockRepository_CompletePresign_Call struct {
*mock.Call
}
// CompletePresign is a helper method to define mock.On call
// - ctx context.Context
// - ownerID uuid.UUID
// - attachmentID uuid.UUID
// - verifiedKey string
// - finalKey string
// - pipelineVersion int
// - maxAttempts int
// - jobID uuid.UUID
func (_e *MockRepository_Expecter) CompletePresign(ctx any, ownerID any, attachmentID any, verifiedKey any, finalKey any, pipelineVersion any, maxAttempts any, jobID any) *MockRepository_CompletePresign_Call {
return &MockRepository_CompletePresign_Call{Call: _e.mock.On("CompletePresign", ctx, ownerID, attachmentID, verifiedKey, finalKey, pipelineVersion, maxAttempts, jobID)}
}
func (_c *MockRepository_CompletePresign_Call) Run(run func(ctx context.Context, ownerID uuid.UUID, attachmentID uuid.UUID, verifiedKey string, finalKey string, pipelineVersion int, maxAttempts int, jobID uuid.UUID)) *MockRepository_CompletePresign_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 uuid.UUID
if args[1] != nil {
arg1 = args[1].(uuid.UUID)
}
var arg2 uuid.UUID
if args[2] != nil {
arg2 = args[2].(uuid.UUID)
}
var arg3 string
if args[3] != nil {
arg3 = args[3].(string)
}
var arg4 string
if args[4] != nil {
arg4 = args[4].(string)
}
var arg5 int
if args[5] != nil {
arg5 = args[5].(int)
}
var arg6 int
if args[6] != nil {
arg6 = args[6].(int)
}
var arg7 uuid.UUID
if args[7] != nil {
arg7 = args[7].(uuid.UUID)
}
run(
arg0,
arg1,
arg2,
arg3,
arg4,
arg5,
arg6,
arg7,
)
})
return _c
}
func (_c *MockRepository_CompletePresign_Call) Return(attachment domain.Attachment, err error) *MockRepository_CompletePresign_Call {
_c.Call.Return(attachment, err)
return _c
}
func (_c *MockRepository_Create_Call) RunAndReturn(run func(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error)) *MockRepository_Create_Call {
func (_c *MockRepository_CompletePresign_Call) RunAndReturn(run func(ctx context.Context, ownerID uuid.UUID, attachmentID uuid.UUID, verifiedKey string, finalKey string, pipelineVersion int, maxAttempts int, jobID uuid.UUID) (domain.Attachment, error)) *MockRepository_CompletePresign_Call {
_c.Call.Return(run)
return _c
}
// CreatePresign provides a mock function for the type MockRepository
func (_mock *MockRepository) CreatePresign(ctx context.Context, draft domain.AttachmentDraft, limits domain.AdmissionLimits) (domain.Attachment, error) {
ret := _mock.Called(ctx, draft, limits)
if len(ret) == 0 {
panic("no return value specified for CreatePresign")
}
var r0 domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.AdmissionLimits) (domain.Attachment, error)); ok {
return returnFunc(ctx, draft, limits)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.AdmissionLimits) domain.Attachment); ok {
r0 = returnFunc(ctx, draft, limits)
} else {
r0 = ret.Get(0).(domain.Attachment)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, domain.AttachmentDraft, domain.AdmissionLimits) error); ok {
r1 = returnFunc(ctx, draft, limits)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_CreatePresign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreatePresign'
type MockRepository_CreatePresign_Call struct {
*mock.Call
}
// CreatePresign is a helper method to define mock.On call
// - ctx context.Context
// - draft domain.AttachmentDraft
// - limits domain.AdmissionLimits
func (_e *MockRepository_Expecter) CreatePresign(ctx any, draft any, limits any) *MockRepository_CreatePresign_Call {
return &MockRepository_CreatePresign_Call{Call: _e.mock.On("CreatePresign", ctx, draft, limits)}
}
func (_c *MockRepository_CreatePresign_Call) Run(run func(ctx context.Context, draft domain.AttachmentDraft, limits domain.AdmissionLimits)) *MockRepository_CreatePresign_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 domain.AttachmentDraft
if args[1] != nil {
arg1 = args[1].(domain.AttachmentDraft)
}
var arg2 domain.AdmissionLimits
if args[2] != nil {
arg2 = args[2].(domain.AdmissionLimits)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockRepository_CreatePresign_Call) Return(attachment domain.Attachment, err error) *MockRepository_CreatePresign_Call {
_c.Call.Return(attachment, err)
return _c
}
func (_c *MockRepository_CreatePresign_Call) RunAndReturn(run func(ctx context.Context, draft domain.AttachmentDraft, limits domain.AdmissionLimits) (domain.Attachment, error)) *MockRepository_CreatePresign_Call {
_c.Call.Return(run)
return _c
}
// CreateQueued provides a mock function for the type MockRepository
func (_mock *MockRepository) CreateQueued(ctx context.Context, draft domain.AttachmentDraft, job domain.Job, limits domain.AdmissionLimits) (domain.Attachment, error) {
ret := _mock.Called(ctx, draft, job, limits)
if len(ret) == 0 {
panic("no return value specified for CreateQueued")
}
var r0 domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.Job, domain.AdmissionLimits) (domain.Attachment, error)); ok {
return returnFunc(ctx, draft, job, limits)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.AttachmentDraft, domain.Job, domain.AdmissionLimits) domain.Attachment); ok {
r0 = returnFunc(ctx, draft, job, limits)
} else {
r0 = ret.Get(0).(domain.Attachment)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, domain.AttachmentDraft, domain.Job, domain.AdmissionLimits) error); ok {
r1 = returnFunc(ctx, draft, job, limits)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_CreateQueued_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateQueued'
type MockRepository_CreateQueued_Call struct {
*mock.Call
}
// CreateQueued is a helper method to define mock.On call
// - ctx context.Context
// - draft domain.AttachmentDraft
// - job domain.Job
// - limits domain.AdmissionLimits
func (_e *MockRepository_Expecter) CreateQueued(ctx any, draft any, job any, limits any) *MockRepository_CreateQueued_Call {
return &MockRepository_CreateQueued_Call{Call: _e.mock.On("CreateQueued", ctx, draft, job, limits)}
}
func (_c *MockRepository_CreateQueued_Call) Run(run func(ctx context.Context, draft domain.AttachmentDraft, job domain.Job, limits domain.AdmissionLimits)) *MockRepository_CreateQueued_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 domain.AttachmentDraft
if args[1] != nil {
arg1 = args[1].(domain.AttachmentDraft)
}
var arg2 domain.Job
if args[2] != nil {
arg2 = args[2].(domain.Job)
}
var arg3 domain.AdmissionLimits
if args[3] != nil {
arg3 = args[3].(domain.AdmissionLimits)
}
run(
arg0,
arg1,
arg2,
arg3,
)
})
return _c
}
func (_c *MockRepository_CreateQueued_Call) Return(attachment domain.Attachment, err error) *MockRepository_CreateQueued_Call {
_c.Call.Return(attachment, err)
return _c
}
func (_c *MockRepository_CreateQueued_Call) RunAndReturn(run func(ctx context.Context, draft domain.AttachmentDraft, job domain.Job, limits domain.AdmissionLimits) (domain.Attachment, error)) *MockRepository_CreateQueued_Call {
_c.Call.Return(run)
return _c
}
@@ -316,6 +579,84 @@ func (_c *MockRepository_GetForOwner_Call) RunAndReturn(run func(ctx context.Con
return _c
}
// GetVariant provides a mock function for the type MockRepository
func (_mock *MockRepository) GetVariant(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int, name domain.VariantName) (domain.Variant, error) {
ret := _mock.Called(ctx, attachmentID, pipelineVersion, name)
if len(ret) == 0 {
panic("no return value specified for GetVariant")
}
var r0 domain.Variant
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int, domain.VariantName) (domain.Variant, error)); ok {
return returnFunc(ctx, attachmentID, pipelineVersion, name)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int, domain.VariantName) domain.Variant); ok {
r0 = returnFunc(ctx, attachmentID, pipelineVersion, name)
} else {
r0 = ret.Get(0).(domain.Variant)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, int, domain.VariantName) error); ok {
r1 = returnFunc(ctx, attachmentID, pipelineVersion, name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_GetVariant_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetVariant'
type MockRepository_GetVariant_Call struct {
*mock.Call
}
// GetVariant is a helper method to define mock.On call
// - ctx context.Context
// - attachmentID uuid.UUID
// - pipelineVersion int
// - name domain.VariantName
func (_e *MockRepository_Expecter) GetVariant(ctx any, attachmentID any, pipelineVersion any, name any) *MockRepository_GetVariant_Call {
return &MockRepository_GetVariant_Call{Call: _e.mock.On("GetVariant", ctx, attachmentID, pipelineVersion, name)}
}
func (_c *MockRepository_GetVariant_Call) Run(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int, name domain.VariantName)) *MockRepository_GetVariant_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 uuid.UUID
if args[1] != nil {
arg1 = args[1].(uuid.UUID)
}
var arg2 int
if args[2] != nil {
arg2 = args[2].(int)
}
var arg3 domain.VariantName
if args[3] != nil {
arg3 = args[3].(domain.VariantName)
}
run(
arg0,
arg1,
arg2,
arg3,
)
})
return _c
}
func (_c *MockRepository_GetVariant_Call) Return(variant domain.Variant, err error) *MockRepository_GetVariant_Call {
_c.Call.Return(variant, err)
return _c
}
func (_c *MockRepository_GetVariant_Call) RunAndReturn(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int, name domain.VariantName) (domain.Variant, error)) *MockRepository_GetVariant_Call {
_c.Call.Return(run)
return _c
}
// IsChatMember provides a mock function for the type MockRepository
func (_mock *MockRepository) IsChatMember(ctx context.Context, chatID uuid.UUID, userID uuid.UUID) (bool, error) {
ret := _mock.Called(ctx, chatID, userID)
@@ -388,196 +729,48 @@ func (_c *MockRepository_IsChatMember_Call) RunAndReturn(run func(ctx context.Co
return _c
}
// ListKeys provides a mock function for the type MockRepository
func (_mock *MockRepository) ListKeys(ctx context.Context, limit int32, offset int32) ([]domain.AttachmentKey, error) {
ret := _mock.Called(ctx, limit, offset)
// ListVariants provides a mock function for the type MockRepository
func (_mock *MockRepository) ListVariants(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int) ([]domain.Variant, error) {
ret := _mock.Called(ctx, attachmentID, pipelineVersion)
if len(ret) == 0 {
panic("no return value specified for ListKeys")
panic("no return value specified for ListVariants")
}
var r0 []domain.AttachmentKey
var r0 []domain.Variant
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, int32, int32) ([]domain.AttachmentKey, error)); ok {
return returnFunc(ctx, limit, offset)
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int) ([]domain.Variant, error)); ok {
return returnFunc(ctx, attachmentID, pipelineVersion)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, int32, int32) []domain.AttachmentKey); ok {
r0 = returnFunc(ctx, limit, offset)
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, int) []domain.Variant); ok {
r0 = returnFunc(ctx, attachmentID, pipelineVersion)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]domain.AttachmentKey)
r0 = ret.Get(0).([]domain.Variant)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, int32, int32) error); ok {
r1 = returnFunc(ctx, limit, offset)
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, int) error); ok {
r1 = returnFunc(ctx, attachmentID, pipelineVersion)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_ListKeys_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListKeys'
type MockRepository_ListKeys_Call struct {
// MockRepository_ListVariants_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListVariants'
type MockRepository_ListVariants_Call struct {
*mock.Call
}
// ListKeys is a helper method to define mock.On call
// ListVariants is a helper method to define mock.On call
// - ctx context.Context
// - limit int32
// - offset int32
func (_e *MockRepository_Expecter) ListKeys(ctx any, limit any, offset any) *MockRepository_ListKeys_Call {
return &MockRepository_ListKeys_Call{Call: _e.mock.On("ListKeys", ctx, limit, offset)}
// - attachmentID uuid.UUID
// - pipelineVersion int
func (_e *MockRepository_Expecter) ListVariants(ctx any, attachmentID any, pipelineVersion any) *MockRepository_ListVariants_Call {
return &MockRepository_ListVariants_Call{Call: _e.mock.On("ListVariants", ctx, attachmentID, pipelineVersion)}
}
func (_c *MockRepository_ListKeys_Call) Run(run func(ctx context.Context, limit int32, offset int32)) *MockRepository_ListKeys_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 int32
if args[1] != nil {
arg1 = args[1].(int32)
}
var arg2 int32
if args[2] != nil {
arg2 = args[2].(int32)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockRepository_ListKeys_Call) Return(attachmentKeys []domain.AttachmentKey, err error) *MockRepository_ListKeys_Call {
_c.Call.Return(attachmentKeys, err)
return _c
}
func (_c *MockRepository_ListKeys_Call) RunAndReturn(run func(ctx context.Context, limit int32, offset int32) ([]domain.AttachmentKey, error)) *MockRepository_ListKeys_Call {
_c.Call.Return(run)
return _c
}
// ListStalePending provides a mock function for the type MockRepository
func (_mock *MockRepository) ListStalePending(ctx context.Context, createdBefore time.Time, limit int32) ([]domain.Attachment, error) {
ret := _mock.Called(ctx, createdBefore, limit)
if len(ret) == 0 {
panic("no return value specified for ListStalePending")
}
var r0 []domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int32) ([]domain.Attachment, error)); ok {
return returnFunc(ctx, createdBefore, limit)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int32) []domain.Attachment); ok {
r0 = returnFunc(ctx, createdBefore, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]domain.Attachment)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, time.Time, int32) error); ok {
r1 = returnFunc(ctx, createdBefore, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_ListStalePending_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListStalePending'
type MockRepository_ListStalePending_Call struct {
*mock.Call
}
// ListStalePending is a helper method to define mock.On call
// - ctx context.Context
// - createdBefore time.Time
// - limit int32
func (_e *MockRepository_Expecter) ListStalePending(ctx any, createdBefore any, limit any) *MockRepository_ListStalePending_Call {
return &MockRepository_ListStalePending_Call{Call: _e.mock.On("ListStalePending", ctx, createdBefore, limit)}
}
func (_c *MockRepository_ListStalePending_Call) Run(run func(ctx context.Context, createdBefore time.Time, limit int32)) *MockRepository_ListStalePending_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 time.Time
if args[1] != nil {
arg1 = args[1].(time.Time)
}
var arg2 int32
if args[2] != nil {
arg2 = args[2].(int32)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockRepository_ListStalePending_Call) Return(attachments []domain.Attachment, err error) *MockRepository_ListStalePending_Call {
_c.Call.Return(attachments, err)
return _c
}
func (_c *MockRepository_ListStalePending_Call) RunAndReturn(run func(ctx context.Context, createdBefore time.Time, limit int32) ([]domain.Attachment, error)) *MockRepository_ListStalePending_Call {
_c.Call.Return(run)
return _c
}
// ListStates provides a mock function for the type MockRepository
func (_mock *MockRepository) ListStates(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID) ([]domain.AttachmentState, error) {
ret := _mock.Called(ctx, chatID, ids)
if len(ret) == 0 {
panic("no return value specified for ListStates")
}
var r0 []domain.AttachmentState
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, []uuid.UUID) ([]domain.AttachmentState, error)); ok {
return returnFunc(ctx, chatID, ids)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, []uuid.UUID) []domain.AttachmentState); ok {
r0 = returnFunc(ctx, chatID, ids)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]domain.AttachmentState)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, []uuid.UUID) error); ok {
r1 = returnFunc(ctx, chatID, ids)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_ListStates_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListStates'
type MockRepository_ListStates_Call struct {
*mock.Call
}
// ListStates is a helper method to define mock.On call
// - ctx context.Context
// - chatID uuid.UUID
// - ids []uuid.UUID
func (_e *MockRepository_Expecter) ListStates(ctx any, chatID any, ids any) *MockRepository_ListStates_Call {
return &MockRepository_ListStates_Call{Call: _e.mock.On("ListStates", ctx, chatID, ids)}
}
func (_c *MockRepository_ListStates_Call) Run(run func(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID)) *MockRepository_ListStates_Call {
func (_c *MockRepository_ListVariants_Call) Run(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int)) *MockRepository_ListVariants_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
@@ -587,9 +780,9 @@ func (_c *MockRepository_ListStates_Call) Run(run func(ctx context.Context, chat
if args[1] != nil {
arg1 = args[1].(uuid.UUID)
}
var arg2 []uuid.UUID
var arg2 int
if args[2] != nil {
arg2 = args[2].([]uuid.UUID)
arg2 = args[2].(int)
}
run(
arg0,
@@ -600,84 +793,12 @@ func (_c *MockRepository_ListStates_Call) Run(run func(ctx context.Context, chat
return _c
}
func (_c *MockRepository_ListStates_Call) Return(attachmentStates []domain.AttachmentState, err error) *MockRepository_ListStates_Call {
_c.Call.Return(attachmentStates, err)
func (_c *MockRepository_ListVariants_Call) Return(variants []domain.Variant, err error) *MockRepository_ListVariants_Call {
_c.Call.Return(variants, err)
return _c
}
func (_c *MockRepository_ListStates_Call) RunAndReturn(run func(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID) ([]domain.AttachmentState, error)) *MockRepository_ListStates_Call {
_c.Call.Return(run)
return _c
}
// MarkReady provides a mock function for the type MockRepository
func (_mock *MockRepository) MarkReady(ctx context.Context, id uuid.UUID, objectKey string) (domain.Attachment, error) {
ret := _mock.Called(ctx, id, objectKey)
if len(ret) == 0 {
panic("no return value specified for MarkReady")
}
var r0 domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, string) (domain.Attachment, error)); ok {
return returnFunc(ctx, id, objectKey)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, string) domain.Attachment); ok {
r0 = returnFunc(ctx, id, objectKey)
} else {
r0 = ret.Get(0).(domain.Attachment)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, string) error); ok {
r1 = returnFunc(ctx, id, objectKey)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepository_MarkReady_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkReady'
type MockRepository_MarkReady_Call struct {
*mock.Call
}
// MarkReady is a helper method to define mock.On call
// - ctx context.Context
// - id uuid.UUID
// - objectKey string
func (_e *MockRepository_Expecter) MarkReady(ctx any, id any, objectKey any) *MockRepository_MarkReady_Call {
return &MockRepository_MarkReady_Call{Call: _e.mock.On("MarkReady", ctx, id, objectKey)}
}
func (_c *MockRepository_MarkReady_Call) Run(run func(ctx context.Context, id uuid.UUID, objectKey string)) *MockRepository_MarkReady_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 uuid.UUID
if args[1] != nil {
arg1 = args[1].(uuid.UUID)
}
var arg2 string
if args[2] != nil {
arg2 = args[2].(string)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockRepository_MarkReady_Call) Return(attachment domain.Attachment, err error) *MockRepository_MarkReady_Call {
_c.Call.Return(attachment, err)
return _c
}
func (_c *MockRepository_MarkReady_Call) RunAndReturn(run func(ctx context.Context, id uuid.UUID, objectKey string) (domain.Attachment, error)) *MockRepository_MarkReady_Call {
func (_c *MockRepository_ListVariants_Call) RunAndReturn(run func(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int) ([]domain.Variant, error)) *MockRepository_ListVariants_Call {
_c.Call.Return(run)
return _c
}
@@ -0,0 +1,398 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
"context"
"mtusi-messenger/internal/domain"
"time"
"github.com/google/uuid"
mock "github.com/stretchr/testify/mock"
)
// NewMockSweepRepository creates a new instance of MockSweepRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockSweepRepository(t interface {
mock.TestingT
Cleanup(func())
}) *MockSweepRepository {
if helper, ok := t.(interface{ Helper() }); ok {
helper.Helper()
}
mock := &MockSweepRepository{}
mock.Mock.Test(t)
t.Cleanup(func() {
if helper, ok := t.(interface{ Helper() }); ok {
helper.Helper()
}
mock.AssertExpectations(t)
})
return mock
}
// MockSweepRepository is an autogenerated mock type for the SweepRepository type
type MockSweepRepository struct {
mock.Mock
}
type MockSweepRepository_Expecter struct {
mock *mock.Mock
}
func (_m *MockSweepRepository) EXPECT() *MockSweepRepository_Expecter {
return &MockSweepRepository_Expecter{mock: &_m.Mock}
}
// DeletePending provides a mock function for the type MockSweepRepository
func (_mock *MockSweepRepository) DeletePending(ctx context.Context, id uuid.UUID) error {
ret := _mock.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for DeletePending")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID) error); ok {
r0 = returnFunc(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockSweepRepository_DeletePending_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeletePending'
type MockSweepRepository_DeletePending_Call struct {
*mock.Call
}
// DeletePending is a helper method to define mock.On call
// - ctx context.Context
// - id uuid.UUID
func (_e *MockSweepRepository_Expecter) DeletePending(ctx any, id any) *MockSweepRepository_DeletePending_Call {
return &MockSweepRepository_DeletePending_Call{Call: _e.mock.On("DeletePending", ctx, id)}
}
func (_c *MockSweepRepository_DeletePending_Call) Run(run func(ctx context.Context, id uuid.UUID)) *MockSweepRepository_DeletePending_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 uuid.UUID
if args[1] != nil {
arg1 = args[1].(uuid.UUID)
}
run(
arg0,
arg1,
)
})
return _c
}
func (_c *MockSweepRepository_DeletePending_Call) Return(err error) *MockSweepRepository_DeletePending_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockSweepRepository_DeletePending_Call) RunAndReturn(run func(ctx context.Context, id uuid.UUID) error) *MockSweepRepository_DeletePending_Call {
_c.Call.Return(run)
return _c
}
// ListPendingPresign provides a mock function for the type MockSweepRepository
func (_mock *MockSweepRepository) ListPendingPresign(ctx context.Context, createdBefore time.Time, limit int) ([]domain.Attachment, error) {
ret := _mock.Called(ctx, createdBefore, limit)
if len(ret) == 0 {
panic("no return value specified for ListPendingPresign")
}
var r0 []domain.Attachment
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int) ([]domain.Attachment, error)); ok {
return returnFunc(ctx, createdBefore, limit)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, int) []domain.Attachment); ok {
r0 = returnFunc(ctx, createdBefore, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]domain.Attachment)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, time.Time, int) error); ok {
r1 = returnFunc(ctx, createdBefore, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockSweepRepository_ListPendingPresign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListPendingPresign'
type MockSweepRepository_ListPendingPresign_Call struct {
*mock.Call
}
// ListPendingPresign is a helper method to define mock.On call
// - ctx context.Context
// - createdBefore time.Time
// - limit int
func (_e *MockSweepRepository_Expecter) ListPendingPresign(ctx any, createdBefore any, limit any) *MockSweepRepository_ListPendingPresign_Call {
return &MockSweepRepository_ListPendingPresign_Call{Call: _e.mock.On("ListPendingPresign", ctx, createdBefore, limit)}
}
func (_c *MockSweepRepository_ListPendingPresign_Call) Run(run func(ctx context.Context, createdBefore time.Time, limit int)) *MockSweepRepository_ListPendingPresign_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 time.Time
if args[1] != nil {
arg1 = args[1].(time.Time)
}
var arg2 int
if args[2] != nil {
arg2 = args[2].(int)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockSweepRepository_ListPendingPresign_Call) Return(attachments []domain.Attachment, err error) *MockSweepRepository_ListPendingPresign_Call {
_c.Call.Return(attachments, err)
return _c
}
func (_c *MockSweepRepository_ListPendingPresign_Call) RunAndReturn(run func(ctx context.Context, createdBefore time.Time, limit int) ([]domain.Attachment, error)) *MockSweepRepository_ListPendingPresign_Call {
_c.Call.Return(run)
return _c
}
// ListReferencedKeys provides a mock function for the type MockSweepRepository
func (_mock *MockSweepRepository) ListReferencedKeys(ctx context.Context, afterKey string, limit int) ([]string, error) {
ret := _mock.Called(ctx, afterKey, limit)
if len(ret) == 0 {
panic("no return value specified for ListReferencedKeys")
}
var r0 []string
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, int) ([]string, error)); ok {
return returnFunc(ctx, afterKey, limit)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, string, int) []string); ok {
r0 = returnFunc(ctx, afterKey, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, string, int) error); ok {
r1 = returnFunc(ctx, afterKey, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockSweepRepository_ListReferencedKeys_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListReferencedKeys'
type MockSweepRepository_ListReferencedKeys_Call struct {
*mock.Call
}
// ListReferencedKeys is a helper method to define mock.On call
// - ctx context.Context
// - afterKey string
// - limit int
func (_e *MockSweepRepository_Expecter) ListReferencedKeys(ctx any, afterKey any, limit any) *MockSweepRepository_ListReferencedKeys_Call {
return &MockSweepRepository_ListReferencedKeys_Call{Call: _e.mock.On("ListReferencedKeys", ctx, afterKey, limit)}
}
func (_c *MockSweepRepository_ListReferencedKeys_Call) Run(run func(ctx context.Context, afterKey string, limit int)) *MockSweepRepository_ListReferencedKeys_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 string
if args[1] != nil {
arg1 = args[1].(string)
}
var arg2 int
if args[2] != nil {
arg2 = args[2].(int)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockSweepRepository_ListReferencedKeys_Call) Return(strings []string, err error) *MockSweepRepository_ListReferencedKeys_Call {
_c.Call.Return(strings, err)
return _c
}
func (_c *MockSweepRepository_ListReferencedKeys_Call) RunAndReturn(run func(ctx context.Context, afterKey string, limit int) ([]string, error)) *MockSweepRepository_ListReferencedKeys_Call {
_c.Call.Return(run)
return _c
}
// ListStaleStaging provides a mock function for the type MockSweepRepository
func (_mock *MockSweepRepository) ListStaleStaging(ctx context.Context, readyBefore time.Time, failedBefore time.Time, limit int) ([]domain.StaleStaging, error) {
ret := _mock.Called(ctx, readyBefore, failedBefore, limit)
if len(ret) == 0 {
panic("no return value specified for ListStaleStaging")
}
var r0 []domain.StaleStaging
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, time.Time, int) ([]domain.StaleStaging, error)); ok {
return returnFunc(ctx, readyBefore, failedBefore, limit)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, time.Time, time.Time, int) []domain.StaleStaging); ok {
r0 = returnFunc(ctx, readyBefore, failedBefore, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]domain.StaleStaging)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, time.Time, time.Time, int) error); ok {
r1 = returnFunc(ctx, readyBefore, failedBefore, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockSweepRepository_ListStaleStaging_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListStaleStaging'
type MockSweepRepository_ListStaleStaging_Call struct {
*mock.Call
}
// ListStaleStaging is a helper method to define mock.On call
// - ctx context.Context
// - readyBefore time.Time
// - failedBefore time.Time
// - limit int
func (_e *MockSweepRepository_Expecter) ListStaleStaging(ctx any, readyBefore any, failedBefore any, limit any) *MockSweepRepository_ListStaleStaging_Call {
return &MockSweepRepository_ListStaleStaging_Call{Call: _e.mock.On("ListStaleStaging", ctx, readyBefore, failedBefore, limit)}
}
func (_c *MockSweepRepository_ListStaleStaging_Call) Run(run func(ctx context.Context, readyBefore time.Time, failedBefore time.Time, limit int)) *MockSweepRepository_ListStaleStaging_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 time.Time
if args[1] != nil {
arg1 = args[1].(time.Time)
}
var arg2 time.Time
if args[2] != nil {
arg2 = args[2].(time.Time)
}
var arg3 int
if args[3] != nil {
arg3 = args[3].(int)
}
run(
arg0,
arg1,
arg2,
arg3,
)
})
return _c
}
func (_c *MockSweepRepository_ListStaleStaging_Call) Return(staleStagings []domain.StaleStaging, err error) *MockSweepRepository_ListStaleStaging_Call {
_c.Call.Return(staleStagings, err)
return _c
}
func (_c *MockSweepRepository_ListStaleStaging_Call) RunAndReturn(run func(ctx context.Context, readyBefore time.Time, failedBefore time.Time, limit int) ([]domain.StaleStaging, error)) *MockSweepRepository_ListStaleStaging_Call {
_c.Call.Return(run)
return _c
}
// MarkStagingDeleted provides a mock function for the type MockSweepRepository
func (_mock *MockSweepRepository) MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error {
ret := _mock.Called(ctx, attachmentID, stagingKey)
if len(ret) == 0 {
panic("no return value specified for MarkStagingDeleted")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, string) error); ok {
r0 = returnFunc(ctx, attachmentID, stagingKey)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockSweepRepository_MarkStagingDeleted_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkStagingDeleted'
type MockSweepRepository_MarkStagingDeleted_Call struct {
*mock.Call
}
// MarkStagingDeleted is a helper method to define mock.On call
// - ctx context.Context
// - attachmentID uuid.UUID
// - stagingKey string
func (_e *MockSweepRepository_Expecter) MarkStagingDeleted(ctx any, attachmentID any, stagingKey any) *MockSweepRepository_MarkStagingDeleted_Call {
return &MockSweepRepository_MarkStagingDeleted_Call{Call: _e.mock.On("MarkStagingDeleted", ctx, attachmentID, stagingKey)}
}
func (_c *MockSweepRepository_MarkStagingDeleted_Call) Run(run func(ctx context.Context, attachmentID uuid.UUID, stagingKey string)) *MockSweepRepository_MarkStagingDeleted_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 uuid.UUID
if args[1] != nil {
arg1 = args[1].(uuid.UUID)
}
var arg2 string
if args[2] != nil {
arg2 = args[2].(string)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *MockSweepRepository_MarkStagingDeleted_Call) Return(err error) *MockSweepRepository_MarkStagingDeleted_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockSweepRepository_MarkStagingDeleted_Call) RunAndReturn(run func(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error) *MockSweepRepository_MarkStagingDeleted_Call {
_c.Call.Return(run)
return _c
}
+311 -97
View File
@@ -17,6 +17,7 @@ import (
"unicode/utf8"
"github.com/google/uuid"
"golang.org/x/sync/semaphore"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/imagecheck"
@@ -27,32 +28,61 @@ const (
contentTypePNG = "image/png"
contentTypeJPEG = "image/jpeg"
objectPrefixAttachments = "attachments/"
objectPrefixStaging = "staging/"
objectPrefixPresign = "staging/presign/"
objectPrefixAttachments = domain.AttachmentsPrefix
objectPrefixStaging = domain.StagingUploadPrefix
objectPrefixPresign = domain.StagingPresignPrefix
)
var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
// Repository is the persistence port. It is consumer-side so the service can be
// tested without PostgreSQL.
// tested without PostgreSQL. Job/outbox writes are atomic with the attachment
// row inside CreateQueued/CompletePresign.
type Repository interface {
IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error)
Create(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error)
MarkReady(ctx context.Context, id uuid.UUID, objectKey string) (domain.Attachment, error)
DeletePending(ctx context.Context, id uuid.UUID) error
GetForMember(ctx context.Context, userID, id uuid.UUID) (domain.Attachment, error)
GetForOwner(ctx context.Context, ownerID, id uuid.UUID) (domain.Attachment, error)
ListStates(ctx context.Context, chatID uuid.UUID, ids []uuid.UUID) ([]domain.AttachmentState, error)
ListStalePending(ctx context.Context, createdBefore time.Time, limit int32) ([]domain.Attachment, error)
ListKeys(ctx context.Context, limit, offset int32) ([]domain.AttachmentKey, error)
ListVariants(ctx context.Context, attachmentID uuid.UUID, pipelineVersion int) ([]domain.Variant, error)
GetVariant(
ctx context.Context,
attachmentID uuid.UUID,
pipelineVersion int,
name domain.VariantName,
) (domain.Variant, error)
CreateQueued(
ctx context.Context,
draft domain.AttachmentDraft,
job domain.Job,
limits domain.AdmissionLimits,
) (domain.Attachment, error)
CreatePresign(
ctx context.Context,
draft domain.AttachmentDraft,
limits domain.AdmissionLimits,
) (domain.Attachment, error)
CompletePresign(
ctx context.Context,
ownerID, attachmentID uuid.UUID,
verifiedKey, finalKey string,
pipelineVersion, maxAttempts int,
jobID uuid.UUID,
) (domain.Attachment, error)
Admission(ctx context.Context, ownerID uuid.UUID) (userPending, totalPending int, stagedBytes int64, err error)
DeletePending(ctx context.Context, id uuid.UUID) error
}
// Config carries the attachment limits. They are validated by config.Load.
// Config carries attachment limits and the shared job contract knobs.
type Config struct {
MaxUploadBytes int64
MaxImagePixels int64
PresignTTL time.Duration
PipelineVersion int
MaxAttempts int
MaxPendingPerUser int
MaxPending int
MaxStagedBytes int64
MaxConcurrent int
}
type Service struct {
@@ -60,6 +90,7 @@ type Service struct {
store objectstore.Store
config Config
log *slog.Logger
upload *semaphore.Weighted
}
func NewService(repo Repository, store objectstore.Store, config Config, log *slog.Logger) *Service {
@@ -67,7 +98,12 @@ func NewService(repo Repository, store objectstore.Store, config Config, log *sl
log = slog.Default()
}
return &Service{repo: repo, store: store, config: config, log: log}
service := &Service{repo: repo, store: store, config: config, log: log}
if config.MaxConcurrent > 0 {
service.upload = semaphore.NewWeighted(int64(config.MaxConcurrent))
}
return service
}
// UploadInput is one raw-body upload. DeclaredContentType comes from the HTTP
@@ -80,7 +116,16 @@ type UploadInput struct {
Body io.Reader
}
// Upload durably stages the original, records the job intent and returns the
// queued snapshot. It never waits for the broker or the worker.
func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachment, error) {
if s.upload != nil {
if err := s.upload.Acquire(ctx, 1); err != nil {
return domain.Attachment{}, unavailable("acquire upload slot", err)
}
defer s.upload.Release(1)
}
if err := ValidateFilename(input.Filename); err != nil {
return domain.Attachment{}, err
}
@@ -94,6 +139,17 @@ func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachm
return domain.Attachment{}, fmt.Errorf("%w: chat is not visible", domain.ErrNotFound)
}
userPending, totalPending, stagedBytes, err := s.repo.Admission(ctx, input.OwnerID)
if err != nil {
return domain.Attachment{}, unavailable("check admission", err)
}
if (s.config.MaxPendingPerUser > 0 && userPending >= s.config.MaxPendingPerUser) ||
(s.config.MaxPending > 0 && totalPending >= s.config.MaxPending) ||
(s.config.MaxStagedBytes > 0 && stagedBytes >= s.config.MaxStagedBytes) {
return domain.Attachment{}, fmt.Errorf("%w: too many unfinished uploads", domain.ErrRateLimited)
}
file, size, sum, err := s.spool(input.Body)
if err != nil {
return domain.Attachment{}, err
@@ -123,79 +179,161 @@ func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachm
return domain.Attachment{}, fmt.Errorf("generate attachment id: %w", err)
}
objectKey := objectPrefixAttachments + input.ChatID.String() + "/" + id.String()
if _, err := s.repo.Create(ctx, domain.AttachmentDraft{
ID: id,
ChatID: input.ChatID,
OwnerID: input.OwnerID,
ObjectKey: objectKey,
Filename: input.Filename,
ContentType: info.ContentType,
SizeBytes: size,
SHA256: sum,
}); err != nil {
return domain.Attachment{}, err
jobID, err := uuid.NewV7()
if err != nil {
return domain.Attachment{}, fmt.Errorf("generate job id: %w", err)
}
stagingKey := domain.StagingUploadKey(input.ChatID, id)
finalKey := domain.OriginalObjectKey(input.ChatID, id)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return domain.Attachment{}, fmt.Errorf("rewind upload: %w", err)
}
if err := s.store.Put(ctx, objectKey, file, size, info.ContentType, map[string]string{"sha256": sum}); err != nil {
s.cleanupPending(ctx, id, "")
return domain.Attachment{}, unavailable("store attachment", err)
if err := s.store.Put(ctx, stagingKey, file, size, info.ContentType, map[string]string{"sha256": sum}); err != nil {
return domain.Attachment{}, unavailable("store staging object", err)
}
ready, err := s.repo.MarkReady(ctx, id, objectKey)
if err := s.verifyStaging(ctx, stagingKey, size, sum); err != nil {
s.deleteObject(ctx, id, stagingKey)
return domain.Attachment{}, unavailable("verify staging object", err)
}
job := domain.Job{
ID: jobID,
AttachmentID: id,
PipelineVersion: s.config.PipelineVersion,
SourceKey: stagingKey,
Attempt: 1,
MaxAttempts: s.config.MaxAttempts,
CreatedAt: time.Now().UTC(),
}
limits := s.admissionLimits()
created, err := s.repo.CreateQueued(ctx, domain.AttachmentDraft{
ID: id,
ChatID: input.ChatID,
OwnerID: input.OwnerID,
ObjectKey: finalKey,
StagingKey: stagingKey,
Filename: input.Filename,
ContentType: info.ContentType,
SizeBytes: size,
SHA256: sum,
Status: domain.AttachmentQueued,
PipelineVersion: s.config.PipelineVersion,
}, job, limits)
if err != nil {
// The object and the pending row remain; the sweeper either finalises
// the row or removes both after the grace period. The client must not
// receive 201 until metadata is committed.
return domain.Attachment{}, unavailable("commit attachment", err)
if errors.Is(err, domain.ErrRateLimited) {
s.deleteObject(ctx, id, stagingKey)
return domain.Attachment{}, err
}
// An ambiguous COMMIT may have persisted the intent; leave the staging
// object for the sweeper instead of destroying a queued job's source.
return domain.Attachment{}, unavailable("commit attachment intent", err)
}
return ready, nil
created.Variants = []domain.Variant{}
return created, nil
}
// Get returns metadata for any public status. Variants are exposed only for the
// currently published pipeline version and only when ready.
func (s *Service) Get(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, error) {
attachment, err := s.repo.GetForMember(ctx, callerID, id)
if err != nil {
return domain.Attachment{}, err
}
// Contract 1.4.0 declares Attachment.status as the single value "ready".
// Pending/failed rows therefore cannot be represented in a 200 response;
// they are reported as 409 until lab5 extends the enum.
if attachment.Status != domain.AttachmentReady {
if !attachment.Status.Public() {
return domain.Attachment{}, fmt.Errorf("%w: attachment is %s", domain.ErrConflict, attachment.Status)
}
return attachment, nil
return s.withVariants(ctx, attachment)
}
// Open returns metadata and the original stream. Only ready attachments have
// bytes; pending/failed return 409 on the content endpoint.
func (s *Service) Open(
// OpenVariant returns metadata plus the selected content stream. Only a ready
// attachment has bytes.
func (s *Service) OpenVariant(
ctx context.Context,
callerID, id uuid.UUID,
) (domain.Attachment, io.ReadCloser, error) {
name domain.VariantName,
) (domain.Attachment, domain.Variant, io.ReadCloser, error) {
attachment, err := s.repo.GetForMember(ctx, callerID, id)
if err != nil {
return domain.Attachment{}, nil, err
return domain.Attachment{}, domain.Variant{}, nil, err
}
if !attachment.Status.Public() || attachment.Status != domain.AttachmentReady {
return domain.Attachment{}, domain.Variant{}, nil, fmt.Errorf(
"%w: attachment is %s", domain.ErrConflict, attachment.Status,
)
}
if name == "" || name == domain.VariantOriginal {
body, _, err := s.store.Open(ctx, attachment.ObjectKey)
if err != nil {
return domain.Attachment{}, domain.Variant{}, nil, err
}
return attachment, domain.Variant{
Name: domain.VariantOriginal,
ContentType: attachment.ContentType,
SizeBytes: attachment.SizeBytes,
SHA256: attachment.SHA256,
ObjectKey: attachment.ObjectKey,
}, body, nil
}
if !name.Valid() {
return domain.Attachment{}, domain.Variant{}, nil, fmt.Errorf(
"%w: variant %q", domain.ErrValidation, name,
)
}
variant, err := s.repo.GetVariant(ctx, attachment.ID, attachment.PipelineVersion, name)
if err != nil {
return domain.Attachment{}, domain.Variant{}, nil, err
}
body, _, err := s.store.Open(ctx, variant.ObjectKey)
if err != nil {
return domain.Attachment{}, domain.Variant{}, nil, err
}
return attachment, variant, body, nil
}
// Open is kept for the legacy service port used by tests.
func (s *Service) Open(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error) {
attachment, _, body, err := s.OpenVariant(ctx, callerID, id, domain.VariantOriginal)
return attachment, body, err
}
func (s *Service) withVariants(ctx context.Context, attachment domain.Attachment) (domain.Attachment, error) {
attachment.Variants = []domain.Variant{}
if attachment.Status != domain.AttachmentReady {
return domain.Attachment{}, nil, fmt.Errorf("%w: attachment is %s", domain.ErrConflict, attachment.Status)
return attachment, nil
}
body, _, err := s.store.Open(ctx, attachment.ObjectKey)
variants, err := s.repo.ListVariants(ctx, attachment.ID, attachment.PipelineVersion)
if err != nil {
return domain.Attachment{}, nil, err
return domain.Attachment{}, err
}
return attachment, body, nil
if variants != nil {
attachment.Variants = variants
}
return attachment, nil
}
func (s *Service) Presign(ctx context.Context, request domain.PresignRequest) (domain.PresignResult, error) {
@@ -235,16 +373,19 @@ func (s *Service) Presign(ctx context.Context, request domain.PresignRequest) (d
stagingKey := objectPrefixPresign + request.ChatID.String() + "/" + id.String()
if _, err := s.repo.Create(ctx, domain.AttachmentDraft{
ID: id,
ChatID: request.ChatID,
OwnerID: request.OwnerID,
ObjectKey: stagingKey,
Filename: request.Filename,
ContentType: request.ContentType,
SizeBytes: request.SizeBytes,
SHA256: request.SHA256,
}); err != nil {
if _, err := s.repo.CreatePresign(ctx, domain.AttachmentDraft{
ID: id,
ChatID: request.ChatID,
OwnerID: request.OwnerID,
ObjectKey: stagingKey,
StagingKey: stagingKey,
Filename: request.Filename,
ContentType: request.ContentType,
SizeBytes: request.SizeBytes,
SHA256: request.SHA256,
Status: domain.AttachmentPending,
PipelineVersion: s.config.PipelineVersion,
}, s.admissionLimits()); err != nil {
return domain.PresignResult{}, err
}
@@ -264,6 +405,10 @@ func (s *Service) Presign(ctx context.Context, request domain.PresignRequest) (d
}, nil
}
// Complete verifies the presigned object, copies the verified bytes to a
// server-owned staging key, and enqueues the same async pipeline as the proxy
// upload. The client's staging key remains writable until the verified copy is
// durable.
func (s *Service) Complete(
ctx context.Context,
ownerID, id uuid.UUID,
@@ -279,12 +424,12 @@ func (s *Service) Complete(
}
switch attachment.Status {
case domain.AttachmentReady:
case domain.AttachmentReady, domain.AttachmentQueued, domain.AttachmentProcessing:
if attachment.SHA256 != requestSHA {
return domain.Attachment{}, fmt.Errorf("%w: attachment already completed with another checksum", domain.ErrConflict)
}
return attachment, nil
return s.withVariants(ctx, attachment)
case domain.AttachmentFailed:
return domain.Attachment{}, fmt.Errorf("%w: attachment is failed", domain.ErrConflict)
case domain.AttachmentPending:
@@ -319,18 +464,31 @@ func (s *Service) Complete(
return domain.Attachment{}, err
}
hasher := sha256.New()
info, validateErr := imagecheck.Validate(io.TeeReader(body, hasher), s.config.MaxImagePixels)
closeErr := body.Close()
file, size, sum, err := s.spool(body)
_ = body.Close()
if validateErr != nil {
s.cleanupPending(ctx, id, attachment.ObjectKey)
return domain.Attachment{}, mapImageError(validateErr)
if err != nil {
return domain.Attachment{}, err
}
if closeErr != nil {
return domain.Attachment{}, unavailable("read uploaded object", closeErr)
defer func() { _ = file.Close() }()
defer func() { _ = os.Remove(file.Name()) }()
if size != attachment.SizeBytes {
s.cleanupPending(ctx, id, attachment.ObjectKey)
return domain.Attachment{}, fmt.Errorf("%w: uploaded object size changed", domain.ErrValidation)
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return domain.Attachment{}, fmt.Errorf("rewind uploaded object: %w", err)
}
info, err := imagecheck.Validate(file, s.config.MaxImagePixels)
if err != nil {
s.cleanupPending(ctx, id, attachment.ObjectKey)
return domain.Attachment{}, mapImageError(err)
}
if info.ContentType != attachment.ContentType {
@@ -342,7 +500,6 @@ func (s *Service) Complete(
)
}
sum := hex.EncodeToString(hasher.Sum(nil))
if sum != attachment.SHA256 {
s.cleanupPending(ctx, id, attachment.ObjectKey)
@@ -350,43 +507,94 @@ func (s *Service) Complete(
}
if requestSHA != attachment.SHA256 {
// The object is valid; only the completion request is inconsistent.
// Keep the pending row and staging object so a retry can succeed.
return domain.Attachment{}, fmt.Errorf("%w: completion checksum mismatch", domain.ErrConflict)
}
finalKey := objectPrefixAttachments + attachment.ChatID.String() + "/" + id.String()
verifiedKey := domain.StagingVerifiedKey(attachment.ChatID, attachment.ID)
finalKey := domain.OriginalObjectKey(attachment.ChatID, attachment.ID)
if err := s.store.Copy(ctx, attachment.ObjectKey, finalKey, attachment.ContentType, map[string]string{"sha256": sum}); err != nil {
return domain.Attachment{}, unavailable("publish attachment", err)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return domain.Attachment{}, fmt.Errorf("rewind verified object: %w", err)
}
ready, err := s.repo.MarkReady(ctx, id, finalKey)
if err := s.store.Put(ctx, verifiedKey, file, size, info.ContentType, map[string]string{"sha256": sum}); err != nil {
return domain.Attachment{}, unavailable("store verified object", err)
}
if err := s.verifyStaging(ctx, verifiedKey, size, sum); err != nil {
s.deleteObject(ctx, id, verifiedKey)
return domain.Attachment{}, unavailable("verify staging object", err)
}
jobID, err := uuid.NewV7()
if err != nil {
// A concurrent completion may have won. Re-read instead of reporting a
// spurious failure.
current, getErr := s.repo.GetForOwner(ctx, ownerID, id)
if getErr == nil && current.Status == domain.AttachmentReady && current.SHA256 == requestSHA {
return current, nil
s.deleteObject(ctx, id, verifiedKey)
return domain.Attachment{}, fmt.Errorf("generate job id: %w", err)
}
queued, err := s.repo.CompletePresign(
ctx, ownerID, id, verifiedKey, finalKey,
s.config.PipelineVersion, s.config.MaxAttempts, jobID,
)
if err != nil {
// The verified key is shared by concurrent completions of this
// attachment. Never delete it here: a winning request may already have
// queued a job pointing at it. The orphan sweeper owns abandoned keys.
if errors.Is(err, domain.ErrConflict) {
current, getErr := s.repo.GetForOwner(ctx, ownerID, id)
if getErr == nil && current.SHA256 == requestSHA && current.Status.Public() {
return s.withVariants(ctx, current)
}
return domain.Attachment{}, fmt.Errorf("%w: attachment is no longer pending", domain.ErrConflict)
}
return domain.Attachment{}, unavailable("commit attachment", err)
return domain.Attachment{}, unavailable("commit presigned attachment", err)
}
if err := s.store.Delete(ctx, attachment.ObjectKey); err != nil {
// The final object is already committed; a stale staging object is an
// orphan and the sweeper removes it after the grace period.
s.log.WarnContext(ctx, "staging object cleanup failed",
if err := s.store.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, domain.ErrNotFound) {
s.log.WarnContext(ctx, "presign staging cleanup failed",
slog.String("attachment_id", id.String()),
slog.Any("error", err),
)
}
return ready, nil
queued.Variants = []domain.Variant{}
return queued, nil
}
// verifyStaging checks the freshly written object; a mismatch is a broken
// staging handoff and must never become a 202.
func (s *Service) admissionLimits() domain.AdmissionLimits {
return domain.AdmissionLimits{
PerUser: s.config.MaxPendingPerUser,
Total: s.config.MaxPending,
Bytes: s.config.MaxStagedBytes,
}
}
func (s *Service) verifyStaging(ctx context.Context, key string, size int64, sum string) error {
head, err := s.store.Head(ctx, key)
if err != nil {
return fmt.Errorf("head staging object: %w", err)
}
if head.Size != size {
return fmt.Errorf("staging size %d, want %d", head.Size, size)
}
if sha := head.Metadata["sha256"]; sha != "" && sha != sum {
return errors.New("staging checksum metadata does not match")
}
return nil
}
func (s *Service) spool(body io.Reader) (*os.File, int64, string, error) {
file, err := os.CreateTemp("", "lab4-upload-*")
file, err := os.CreateTemp("", "lab5-upload-*")
if err != nil {
return nil, 0, "", fmt.Errorf("create upload spool: %w", err)
}
@@ -427,16 +635,22 @@ func (s *Service) spool(body io.Reader) (*os.File, int64, string, error) {
return file, size, hex.EncodeToString(hasher.Sum(nil)), nil
}
func (s *Service) cleanupPending(ctx context.Context, id uuid.UUID, objectKey string) {
if objectKey != "" {
if err := s.store.Delete(ctx, objectKey); err != nil && !errors.Is(err, domain.ErrNotFound) {
s.log.WarnContext(ctx, "object cleanup failed",
slog.String("attachment_id", id.String()),
slog.Any("error", err),
)
}
func (s *Service) deleteObject(ctx context.Context, id uuid.UUID, key string) {
if key == "" {
return
}
if err := s.store.Delete(ctx, key); err != nil && !errors.Is(err, domain.ErrNotFound) {
s.log.WarnContext(ctx, "object cleanup failed",
slog.String("attachment_id", id.String()),
slog.Any("error", err),
)
}
}
func (s *Service) cleanupPending(ctx context.Context, id uuid.UUID, objectKey string) {
s.deleteObject(ctx, id, objectKey)
if err := s.repo.DeletePending(ctx, id); err != nil && !errors.Is(err, domain.ErrNotFound) {
s.log.WarnContext(ctx, "pending attachment cleanup failed",
slog.String("attachment_id", id.String()),
+197 -49
View File
@@ -71,6 +71,8 @@ func (f *fakeStore) Open(_ context.Context, key string) (io.ReadCloser, objectst
return io.NopCloser(bytes.NewReader(object.data)), f.object(key, object), nil
}
func (f *fakeStore) Ping(context.Context) error { return nil }
func (f *fakeStore) Head(_ context.Context, key string) (objectstore.Object, error) {
object, ok := f.objects[key]
if !ok {
@@ -144,9 +146,11 @@ func pngBytes(t *testing.T, width, height int) []byte {
func serviceConfig() attachment.Config {
return attachment.Config{
MaxUploadBytes: 1 << 20,
MaxImagePixels: 1_000_000,
PresignTTL: 5 * time.Minute,
MaxUploadBytes: 1 << 20,
MaxImagePixels: 1_000_000,
PresignTTL: 5 * time.Minute,
PipelineVersion: 1,
MaxAttempts: 5,
}
}
@@ -162,7 +166,7 @@ func TestValidateFilename(t *testing.T) {
}
}
func TestUploadStoresOriginal(t *testing.T) {
func TestUploadStagesAndQueues(t *testing.T) {
repo := mocks.NewMockRepository(t)
store := newFakeStore()
service := attachment.NewService(repo, store, serviceConfig(), slog.Default())
@@ -171,16 +175,25 @@ func TestUploadStoresOriginal(t *testing.T) {
ownerID := uuid.New()
content := pngBytes(t, 16, 16)
sum := sha256.Sum256(content)
digest := hex.EncodeToString(sum[:])
repo.EXPECT().IsChatMember(mock.Anything, chatID, ownerID).Return(true, nil).Once()
repo.EXPECT().Create(mock.Anything, mock.MatchedBy(func(draft domain.AttachmentDraft) bool {
repo.EXPECT().Admission(mock.Anything, ownerID).Return(0, 0, int64(0), nil).Once()
repo.EXPECT().CreateQueued(mock.Anything, mock.MatchedBy(func(draft domain.AttachmentDraft) bool {
return draft.ChatID == chatID && draft.OwnerID == ownerID &&
draft.ContentType == "image/png" && draft.SizeBytes == int64(len(content)) &&
draft.SHA256 == hex.EncodeToString(sum[:]) && strings.HasPrefix(draft.ObjectKey, "attachments/")
})).Return(domain.Attachment{Status: domain.AttachmentPending}, nil).Once()
repo.EXPECT().MarkReady(mock.Anything, mock.Anything, mock.Anything).
Return(domain.Attachment{Status: domain.AttachmentReady, ContentType: "image/png", SizeBytes: int64(len(content))}, nil).
Once()
draft.SHA256 == digest && draft.Status == domain.AttachmentQueued &&
draft.PipelineVersion == 1 &&
strings.HasPrefix(draft.ObjectKey, "attachments/") &&
strings.HasPrefix(draft.StagingKey, "staging/uploads/")
}), mock.MatchedBy(func(job domain.Job) bool {
return job.PipelineVersion == 1 && job.Attempt == 1 &&
job.MaxAttempts == 5 && job.SourceKey != ""
}), domain.AdmissionLimits{}).Return(domain.Attachment{
ID: uuid.New(), ChatID: chatID, OwnerID: ownerID,
ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentQueued, PipelineVersion: 1,
}, nil).Once()
result, err := service.Upload(t.Context(), attachment.UploadInput{
ChatID: chatID,
@@ -190,28 +203,28 @@ func TestUploadStoresOriginal(t *testing.T) {
Body: bytes.NewReader(content),
})
require.NoError(t, err)
require.Equal(t, domain.AttachmentReady, result.Status)
require.Equal(t, domain.AttachmentQueued, result.Status)
require.Len(t, store.objects, 1)
for key := range store.objects {
require.Contains(t, key, "staging/uploads/")
}
}
func TestUploadRejectsTooLargeBody(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), attachment.Config{
MaxUploadBytes: 8,
MaxImagePixels: 1_000_000,
PresignTTL: time.Minute,
MaxUploadBytes: 8, MaxImagePixels: 1_000_000, PresignTTL: time.Minute,
PipelineVersion: 1, MaxAttempts: 5,
}, slog.Default())
repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once()
repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(0, 0, int64(0), nil).Once()
_, err := service.Upload(t.Context(), attachment.UploadInput{
ChatID: uuid.New(),
OwnerID: uuid.New(),
Filename: "photo.png",
ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png",
DeclaredContentType: "image/png",
Body: bytes.NewReader(bytes.Repeat([]byte("x"), 9)),
})
require.ErrorIs(t, err, domain.ErrTooLarge)
}
@@ -220,19 +233,63 @@ func TestUploadRejectsMIMEMismatch(t *testing.T) {
service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default())
repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once()
repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(0, 0, int64(0), nil).Once()
_, err := service.Upload(t.Context(), attachment.UploadInput{
ChatID: uuid.New(),
OwnerID: uuid.New(),
Filename: "photo.jpg",
ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.jpg",
DeclaredContentType: "image/jpeg",
Body: bytes.NewReader(pngBytes(t, 8, 8)),
})
require.ErrorIs(t, err, domain.ErrUnsupportedMedia)
}
func TestGetRejectsNonReadyAttachment(t *testing.T) {
func TestUploadAdmissionRejectsFlood(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), attachment.Config{
MaxUploadBytes: 1 << 20, MaxImagePixels: 1_000_000, PresignTTL: time.Minute,
PipelineVersion: 1, MaxAttempts: 5, MaxPendingPerUser: 1,
}, slog.Default())
repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once()
repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(1, 1, int64(0), nil).Once()
_, err := service.Upload(t.Context(), attachment.UploadInput{
ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png",
DeclaredContentType: "image/png", Body: bytes.NewReader(pngBytes(t, 8, 8)),
})
require.ErrorIs(t, err, domain.ErrRateLimited)
}
func TestGetReturnsQueuedSnapshotWithoutVariants(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default())
repo.EXPECT().GetForMember(mock.Anything, mock.Anything, mock.Anything).
Return(domain.Attachment{Status: domain.AttachmentQueued, PipelineVersion: 1}, nil).Once()
got, err := service.Get(t.Context(), uuid.New(), uuid.New())
require.NoError(t, err)
require.Equal(t, domain.AttachmentQueued, got.Status)
require.Empty(t, got.Variants)
}
func TestGetReadyLoadsCurrentVariants(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default())
attachmentID := uuid.New()
variant := domain.Variant{Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 10, Width: 256, Height: 160, SHA256: strings.Repeat("a", 64)}
repo.EXPECT().GetForMember(mock.Anything, mock.Anything, attachmentID).
Return(domain.Attachment{ID: attachmentID, Status: domain.AttachmentReady, PipelineVersion: 2}, nil).Once()
repo.EXPECT().ListVariants(mock.Anything, attachmentID, 2).Return([]domain.Variant{variant}, nil).Once()
got, err := service.Get(t.Context(), uuid.New(), attachmentID)
require.NoError(t, err)
require.Equal(t, []domain.Variant{variant}, got.Variants)
}
func TestGetRejectsInternalPending(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default())
@@ -243,7 +300,18 @@ func TestGetRejectsNonReadyAttachment(t *testing.T) {
require.ErrorIs(t, err, domain.ErrConflict)
}
func TestCompletePublishesPresignedObject(t *testing.T) {
func TestOpenVariantRejectsQueued(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default())
repo.EXPECT().GetForMember(mock.Anything, mock.Anything, mock.Anything).
Return(domain.Attachment{Status: domain.AttachmentQueued}, nil).Once()
_, _, _, err := service.OpenVariant(t.Context(), uuid.New(), uuid.New(), domain.VariantOriginal)
require.ErrorIs(t, err, domain.ErrConflict)
}
func TestCompleteQueuesPresignedObject(t *testing.T) {
repo := mocks.NewMockRepository(t)
store := newFakeStore()
service := attachment.NewService(repo, store, serviceConfig(), slog.Default())
@@ -257,33 +325,119 @@ func TestCompletePublishesPresignedObject(t *testing.T) {
stagingKey := "staging/presign/" + chatID.String() + "/" + id.String()
store.objects[stagingKey] = storedObject{
data: content,
contentType: "image/png",
metadata: map[string]string{"sha256": digest},
lastModified: time.Now().UTC(),
data: content, contentType: "image/png",
metadata: map[string]string{"sha256": digest}, lastModified: time.Now().UTC(),
}
pending := domain.Attachment{
ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey,
ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentPending,
StagingKey: stagingKey, ContentType: "image/png",
SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentPending, PipelineVersion: 1,
}
repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once()
repo.EXPECT().MarkReady(mock.Anything, id, mock.MatchedBy(func(key string) bool {
return strings.HasPrefix(key, "attachments/")
})).Return(domain.Attachment{
repo.EXPECT().CompletePresign(
mock.Anything, ownerID, id,
mock.MatchedBy(func(key string) bool { return strings.HasPrefix(key, "staging/verified/") }),
mock.MatchedBy(func(key string) bool { return strings.HasPrefix(key, "attachments/") }),
1, 5, mock.Anything,
).Return(domain.Attachment{
ID: id, ChatID: chatID, OwnerID: ownerID, ContentType: "image/png",
SizeBytes: int64(len(content)), SHA256: digest, Status: domain.AttachmentReady,
SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentQueued, PipelineVersion: 1,
}, nil).Once()
completed, err := service.Complete(t.Context(), ownerID, id, digest)
require.NoError(t, err)
require.Equal(t, domain.AttachmentReady, completed.Status)
require.Equal(t, domain.AttachmentQueued, completed.Status)
require.NotContains(t, store.objects, stagingKey)
require.Len(t, store.objects, 1)
}
func TestCompleteConcurrentConflictKeepsVerifiedSource(t *testing.T) {
repo := mocks.NewMockRepository(t)
store := newFakeStore()
service := attachment.NewService(repo, store, serviceConfig(), slog.Default())
id := uuid.New()
chatID := uuid.New()
ownerID := uuid.New()
content := pngBytes(t, 10, 10)
sum := sha256.Sum256(content)
digest := hex.EncodeToString(sum[:])
stagingKey := "staging/presign/" + chatID.String() + "/" + id.String()
store.objects[stagingKey] = storedObject{
data: content, contentType: "image/png",
metadata: map[string]string{"sha256": digest},
}
pending := domain.Attachment{
ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey,
StagingKey: stagingKey, ContentType: "image/png",
SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentPending, PipelineVersion: 1,
}
queued := domain.Attachment{
ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: "attachments/" + chatID.String() + "/" + id.String(),
ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentQueued, PipelineVersion: 1,
}
repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once()
repo.EXPECT().CompletePresign(
mock.Anything, ownerID, id, mock.Anything, mock.Anything,
1, 5, mock.Anything,
).Return(domain.Attachment{}, domain.ErrConflict).Once()
repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(queued, nil).Once()
got, err := service.Complete(t.Context(), ownerID, id, digest)
require.NoError(t, err)
require.Equal(t, domain.AttachmentQueued, got.Status)
verified := 0
for key := range store.objects {
if strings.HasPrefix(key, "staging/verified/") {
verified++
}
}
require.Equal(t, 1, verified, "loser must not delete the winner's verified source")
}
func TestPresignHonoursAdmissionLimit(t *testing.T) {
repo := mocks.NewMockRepository(t)
service := attachment.NewService(repo, newFakeStore(), serviceConfig(), slog.Default())
repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once()
repo.EXPECT().CreatePresign(mock.Anything, mock.Anything, mock.Anything).
Return(domain.Attachment{}, domain.ErrRateLimited).Once()
_, err := service.Presign(t.Context(), domain.PresignRequest{
ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png",
ContentType: "image/png", SizeBytes: 10, SHA256: strings.Repeat("a", 64),
})
require.ErrorIs(t, err, domain.ErrRateLimited)
}
func TestUploadRateLimitedByAtomicAdmissionRemovesStaging(t *testing.T) {
repo := mocks.NewMockRepository(t)
store := newFakeStore()
service := attachment.NewService(repo, store, serviceConfig(), slog.Default())
repo.EXPECT().IsChatMember(mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Once()
repo.EXPECT().Admission(mock.Anything, mock.Anything).Return(0, 0, int64(0), nil).Once()
repo.EXPECT().CreateQueued(mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(domain.Attachment{}, domain.ErrRateLimited).Once()
_, err := service.Upload(t.Context(), attachment.UploadInput{
ChatID: uuid.New(), OwnerID: uuid.New(), Filename: "photo.png",
DeclaredContentType: "image/png", Body: bytes.NewReader(pngBytes(t, 8, 8)),
})
require.ErrorIs(t, err, domain.ErrRateLimited)
require.Empty(t, store.objects, "a rejected atomic admission must not leave staging")
}
func TestCompleteRejectsChecksumMismatch(t *testing.T) {
repo := mocks.NewMockRepository(t)
store := newFakeStore()
@@ -297,16 +451,13 @@ func TestCompleteRejectsChecksumMismatch(t *testing.T) {
digest := hex.EncodeToString(sum[:])
stagingKey := "staging/presign/" + chatID.String() + "/" + id.String()
store.objects[stagingKey] = storedObject{
data: content,
contentType: "image/png",
metadata: map[string]string{"sha256": digest},
}
store.objects[stagingKey] = storedObject{data: content, contentType: "image/png", metadata: map[string]string{"sha256": digest}}
pending := domain.Attachment{
ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey,
ContentType: "image/png", SizeBytes: int64(len(content)) + 1, SHA256: digest,
Status: domain.AttachmentPending,
StagingKey: stagingKey, ContentType: "image/png",
SizeBytes: int64(len(content)) + 1, SHA256: digest,
Status: domain.AttachmentPending, PipelineVersion: 1,
}
repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once()
@@ -330,15 +481,12 @@ func TestCompleteKeepsObjectOnWrongRequestChecksum(t *testing.T) {
digest := hex.EncodeToString(sum[:])
stagingKey := "staging/presign/" + chatID.String() + "/" + id.String()
store.objects[stagingKey] = storedObject{
data: content,
contentType: "image/png",
metadata: map[string]string{"sha256": digest},
}
store.objects[stagingKey] = storedObject{data: content, contentType: "image/png", metadata: map[string]string{"sha256": digest}}
pending := domain.Attachment{
ID: id, ChatID: chatID, OwnerID: ownerID, ObjectKey: stagingKey,
ContentType: "image/png", SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentPending,
StagingKey: stagingKey, ContentType: "image/png",
SizeBytes: int64(len(content)), SHA256: digest,
Status: domain.AttachmentPending, PipelineVersion: 1,
}
repo.EXPECT().GetForOwner(mock.Anything, ownerID, id).Return(pending, nil).Once()
+78 -84
View File
@@ -4,9 +4,10 @@ import (
"context"
"errors"
"log/slog"
"strings"
"time"
"github.com/google/uuid"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/platform/objectstore"
)
@@ -16,26 +17,38 @@ type LockStore interface {
AcquireSweepLock(ctx context.Context) (release func(), acquired bool, err error)
}
// SweepRepository is the storage port used by cleanup. It exposes only
// read/delete projections; object bytes still live in S3.
type SweepRepository interface {
ListPendingPresign(ctx context.Context, createdBefore time.Time, limit int) ([]domain.Attachment, error)
DeletePending(ctx context.Context, id uuid.UUID) error
ListStaleStaging(ctx context.Context, readyBefore, failedBefore time.Time, limit int) ([]domain.StaleStaging, error)
MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error
ListReferencedKeys(ctx context.Context, afterKey string, limit int) ([]string, error)
}
type SweepStats struct {
Recovered int
RowsDeleted int
ObjectsDeleted int
StagingCleared int
}
type Sweeper struct {
repo Repository
store objectstore.Store
locker LockStore
grace time.Duration
interval time.Duration
log *slog.Logger
repo SweepRepository
store objectstore.Store
locker LockStore
grace time.Duration
orphanGrace time.Duration
failedRetain time.Duration
interval time.Duration
log *slog.Logger
}
func NewSweeper(
repo Repository,
repo SweepRepository,
store objectstore.Store,
locker LockStore,
grace, interval time.Duration,
grace, orphanGrace, failedRetain, interval time.Duration,
log *slog.Logger,
) *Sweeper {
if log == nil {
@@ -43,12 +56,14 @@ func NewSweeper(
}
return &Sweeper{
repo: repo,
store: store,
locker: locker,
grace: grace,
interval: interval,
log: log,
repo: repo,
store: store,
locker: locker,
grace: grace,
orphanGrace: orphanGrace,
failedRetain: failedRetain,
interval: interval,
log: log,
}
}
@@ -77,18 +92,18 @@ func (s *Sweeper) sweep(ctx context.Context) {
return
}
if stats.Recovered+stats.RowsDeleted+stats.ObjectsDeleted > 0 {
if stats.RowsDeleted+stats.ObjectsDeleted+stats.StagingCleared > 0 {
s.log.InfoContext(ctx, "attachment sweep finished",
slog.Int("recovered", stats.Recovered),
slog.Int("rows_deleted", stats.RowsDeleted),
slog.Int("objects_deleted", stats.ObjectsDeleted),
slog.Int("staging_cleared", stats.StagingCleared),
)
}
}
// SweepOnce reconciles PostgreSQL metadata with S3 objects. It never deletes a
// ready row or an object referenced by any row. Only objects older than the
// grace period are candidates, so an in-flight upload cannot be collected.
// SweepOnce applies explicit, status-aware retention rules. It never deletes
// staging of a queued/processing attachment and aborts object collection if any
// database page or S3 listing fails.
func (s *Sweeper) SweepOnce(ctx context.Context) (SweepStats, error) {
release, acquired, err := s.locker.AcquireSweepLock(ctx)
if err != nil {
@@ -101,72 +116,20 @@ func (s *Sweeper) SweepOnce(ctx context.Context) (SweepStats, error) {
defer release()
before := time.Now().UTC().Add(-s.grace)
now := time.Now().UTC()
var stats SweepStats
stale, err := s.repo.ListStalePending(ctx, before, 100)
// Abandoned presign rows: no completion ever arrived.
pending, err := s.repo.ListPendingPresign(ctx, now.Add(-s.grace), 100)
if err != nil {
return stats, err
}
for _, row := range stale {
if row.Status != domain.AttachmentPending {
continue
}
if strings.HasPrefix(row.ObjectKey, objectPrefixPresign) {
if s.deleteObject(ctx, row.ObjectKey) {
for _, row := range pending {
if row.StagingKey != "" {
if s.deleteObject(ctx, row.StagingKey) {
stats.ObjectsDeleted++
}
if err := s.repo.DeletePending(ctx, row.ID); err != nil && !errors.Is(err, domain.ErrNotFound) {
return stats, err
}
stats.RowsDeleted++
continue
}
// A proxy upload writes directly to its final key. If the process died
// between PutObject and MarkReady, the object is complete and can be
// recovered instead of discarded.
head, err := s.store.Head(ctx, row.ObjectKey)
if err != nil {
if errors.Is(err, domain.ErrNotFound) {
if err := s.repo.DeletePending(ctx, row.ID); err != nil && !errors.Is(err, domain.ErrNotFound) {
return stats, err
}
stats.RowsDeleted++
continue
}
s.log.WarnContext(ctx, "sweep head failed",
slog.String("attachment_id", row.ID.String()),
slog.Any("error", err),
)
continue
}
if head.Size == row.SizeBytes && head.Metadata["sha256"] == row.SHA256 {
if _, err := s.repo.MarkReady(ctx, row.ID, row.ObjectKey); err != nil {
// A transient database failure must not destroy a valid object.
// Leave both the row and the object for the next sweep.
s.log.WarnContext(ctx, "sweep recovery failed",
slog.String("attachment_id", row.ID.String()),
slog.Any("error", err),
)
continue
}
stats.Recovered++
continue
}
if s.deleteObject(ctx, row.ObjectKey) {
stats.ObjectsDeleted++
}
if err := s.repo.DeletePending(ctx, row.ID); err != nil && !errors.Is(err, domain.ErrNotFound) {
@@ -176,12 +139,40 @@ func (s *Sweeper) SweepOnce(ctx context.Context) (SweepStats, error) {
stats.RowsDeleted++
}
// Ready/failed rows keep their staging key until the object is safely
// removed. Active queued/processing staging is never in this list.
stale, err := s.repo.ListStaleStaging(
ctx, now.Add(-s.orphanGrace), now.Add(-s.failedRetain), 100,
)
if err != nil {
return stats, err
}
for _, row := range stale {
if s.deleteObject(ctx, row.StagingKey) {
stats.ObjectsDeleted++
}
if err := s.repo.MarkStagingDeleted(ctx, row.AttachmentID, row.StagingKey); err != nil {
return stats, err
}
stats.StagingCleared++
}
referenced, err := s.referencedKeys(ctx)
if err != nil {
return stats, err
}
for _, prefix := range []string{objectPrefixAttachments, objectPrefixStaging} {
// Orphans: objects no row references. Final objects and variants use the
// normal grace; staging uses the longer orphan grace.
for _, prefix := range []string{domain.AttachmentsPrefix, "staging/"} {
before := now.Add(-s.grace)
if prefix == "staging/" {
before = now.Add(-s.orphanGrace)
}
objects, err := s.store.List(ctx, prefix)
if err != nil {
return stats, err
@@ -209,20 +200,23 @@ func (s *Sweeper) referencedKeys(ctx context.Context) (map[string]struct{}, erro
const pageSize = 1000
keys := make(map[string]struct{})
after := ""
for offset := int32(0); ; offset += pageSize {
rows, err := s.repo.ListKeys(ctx, pageSize, offset)
for {
rows, err := s.repo.ListReferencedKeys(ctx, after, pageSize)
if err != nil {
return nil, err
}
for _, row := range rows {
keys[row.ObjectKey] = struct{}{}
for _, key := range rows {
keys[key] = struct{}{}
}
if len(rows) < pageSize {
return keys, nil
}
after = rows[len(rows)-1]
}
}
+57 -70
View File
@@ -15,73 +15,71 @@ import (
"mtusi-messenger/internal/service/attachment/mocks"
)
func TestSweepReconcilesPendingAndOrphanObjects(t *testing.T) {
repo := mocks.NewMockRepository(t)
func TestSweepAppliesStatusAwareRetention(t *testing.T) {
repo := mocks.NewMockSweepRepository(t)
locker := mocks.NewMockLockStore(t)
store := newFakeStore()
stale := time.Now().UTC().Add(-time.Hour)
stale := time.Now().UTC().Add(-2 * time.Hour)
presign := domain.Attachment{
ID: uuid.New(), ObjectKey: "staging/presign/chat/" + uuid.NewString(),
Status: domain.AttachmentPending, CreatedAt: stale,
ID: uuid.New(), StagingKey: "staging/presign/chat/" + uuid.NewString(),
Status: domain.AttachmentPending,
}
proxyRecoverable := domain.Attachment{
ID: uuid.New(), ObjectKey: "attachments/chat/" + uuid.NewString(),
SizeBytes: 3, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Status: domain.AttachmentPending, CreatedAt: stale,
staleStaging := domain.StaleStaging{
AttachmentID: uuid.New(), StagingKey: "staging/uploads/chat/ready", Status: domain.AttachmentReady,
}
proxyMissing := domain.Attachment{
ID: uuid.New(), ObjectKey: "attachments/chat/" + uuid.NewString(),
Status: domain.AttachmentPending, CreatedAt: stale,
}
readyKey := "attachments/chat/ready"
activeStaging := "staging/uploads/chat/active"
readyKey := "attachments/chat/ready/original"
variantKey := "attachments/chat/ready/variants/v1/thumbnail.jpg"
store.objects[presign.ObjectKey] = storedObject{data: []byte("x"), lastModified: stale}
store.objects[proxyRecoverable.ObjectKey] = storedObject{
data: []byte("abc"),
lastModified: stale,
metadata: map[string]string{"sha256": proxyRecoverable.SHA256},
for _, key := range []string{
presign.StagingKey, staleStaging.StagingKey, activeStaging, readyKey, variantKey,
"attachments/chat/orphan", "staging/orphan",
} {
store.objects[key] = storedObject{data: []byte("x"), lastModified: stale}
}
store.objects[readyKey] = storedObject{data: []byte("ready"), lastModified: stale}
store.objects["attachments/chat/orphan"] = storedObject{data: []byte("orphan"), lastModified: stale}
store.objects["staging/abandoned"] = storedObject{data: []byte("stale"), lastModified: stale}
store.objects["staging/fresh"] = storedObject{data: []byte("fresh"), lastModified: time.Now().UTC()}
store.objects["staging/fresh"] = storedObject{data: []byte("x"), lastModified: time.Now().UTC()}
repo.EXPECT().ListStalePending(mock.Anything, mock.Anything, int32(100)).
Return([]domain.Attachment{presign, proxyRecoverable, proxyMissing}, nil).Once()
repo.EXPECT().ListPendingPresign(mock.Anything, mock.Anything, 100).
Return([]domain.Attachment{presign}, nil).Once()
repo.EXPECT().DeletePending(mock.Anything, presign.ID).Return(nil).Once()
repo.EXPECT().MarkReady(mock.Anything, proxyRecoverable.ID, proxyRecoverable.ObjectKey).
Return(domain.Attachment{ID: proxyRecoverable.ID, Status: domain.AttachmentReady}, nil).Once()
repo.EXPECT().DeletePending(mock.Anything, proxyMissing.ID).Return(nil).Once()
repo.EXPECT().ListKeys(mock.Anything, int32(1000), int32(0)).
Return([]domain.AttachmentKey{
{ID: proxyRecoverable.ID, ObjectKey: proxyRecoverable.ObjectKey, Status: domain.AttachmentReady},
{ID: uuid.New(), ObjectKey: readyKey, Status: domain.AttachmentReady},
}, nil).Once()
locker.EXPECT().AcquireSweepLock(mock.Anything).
Return(func() {}, true, nil).Once()
repo.EXPECT().ListStaleStaging(mock.Anything, mock.Anything, mock.Anything, 100).
Return([]domain.StaleStaging{staleStaging}, nil).Once()
repo.EXPECT().MarkStagingDeleted(mock.Anything, staleStaging.AttachmentID, staleStaging.StagingKey).
Return(nil).Once()
repo.EXPECT().ListReferencedKeys(mock.Anything, "", 1000).
Return([]string{activeStaging, readyKey, variantKey}, nil).Once()
locker.EXPECT().AcquireSweepLock(mock.Anything).Return(func() {}, true, nil).Once()
sweeper := attachment.NewSweeper(repo, store, locker, time.Minute, time.Hour, slog.Default())
sweeper := attachment.NewSweeper(
repo, store, locker,
10*time.Minute, time.Hour, 24*time.Hour, time.Minute, slog.Default(),
)
stats, err := sweeper.SweepOnce(t.Context())
require.NoError(t, err)
require.Equal(t, 1, stats.Recovered)
require.Equal(t, 2, stats.RowsDeleted)
require.Equal(t, 3, stats.ObjectsDeleted)
require.Equal(t, 1, stats.RowsDeleted)
require.Equal(t, 1, stats.StagingCleared)
require.Equal(t, 4, stats.ObjectsDeleted)
require.NotContains(t, store.objects, presign.ObjectKey)
require.NotContains(t, store.objects, presign.StagingKey)
require.NotContains(t, store.objects, staleStaging.StagingKey)
require.NotContains(t, store.objects, "attachments/chat/orphan")
require.NotContains(t, store.objects, "staging/abandoned")
require.Contains(t, store.objects, proxyRecoverable.ObjectKey)
require.NotContains(t, store.objects, "staging/orphan")
require.Contains(t, store.objects, activeStaging)
require.Contains(t, store.objects, readyKey)
require.Contains(t, store.objects, variantKey)
require.Contains(t, store.objects, "staging/fresh")
}
func TestSweepSkipsWhenAnotherReplicaHoldsTheLock(t *testing.T) {
repo := mocks.NewMockRepository(t)
repo := mocks.NewMockSweepRepository(t)
locker := mocks.NewMockLockStore(t)
sweeper := attachment.NewSweeper(repo, newFakeStore(), locker, time.Minute, time.Hour, slog.Default())
sweeper := attachment.NewSweeper(
repo, newFakeStore(), locker,
time.Minute, time.Hour, 24*time.Hour, time.Minute, slog.Default(),
)
locker.EXPECT().AcquireSweepLock(mock.Anything).Return(nil, false, nil).Once()
@@ -90,37 +88,26 @@ func TestSweepSkipsWhenAnotherReplicaHoldsTheLock(t *testing.T) {
require.Zero(t, stats)
}
func TestSweepKeepsObjectWhenMarkReadyFails(t *testing.T) {
repo := mocks.NewMockRepository(t)
func TestSweepAbortsObjectCollectionWhenKeysCannotBeRead(t *testing.T) {
repo := mocks.NewMockSweepRepository(t)
locker := mocks.NewMockLockStore(t)
store := newFakeStore()
stale := time.Now().UTC().Add(-time.Hour)
row := domain.Attachment{
ID: uuid.New(), ObjectKey: "attachments/chat/" + uuid.NewString(),
SizeBytes: 3, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Status: domain.AttachmentPending, CreatedAt: stale,
}
orphan := "attachments/chat/orphan"
store.objects[orphan] = storedObject{data: []byte("x"), lastModified: time.Now().UTC().Add(-2 * time.Hour)}
store.objects[row.ObjectKey] = storedObject{
data: []byte("abc"),
lastModified: stale,
metadata: map[string]string{"sha256": row.SHA256},
}
repo.EXPECT().ListStalePending(mock.Anything, mock.Anything, int32(100)).
Return([]domain.Attachment{row}, nil).Once()
repo.EXPECT().MarkReady(mock.Anything, row.ID, row.ObjectKey).
Return(domain.Attachment{}, context.DeadlineExceeded).Once()
repo.EXPECT().ListKeys(mock.Anything, int32(1000), int32(0)).
Return([]domain.AttachmentKey{{ID: row.ID, ObjectKey: row.ObjectKey, Status: domain.AttachmentPending}}, nil).Once()
repo.EXPECT().ListPendingPresign(mock.Anything, mock.Anything, 100).Return(nil, nil).Once()
repo.EXPECT().ListStaleStaging(mock.Anything, mock.Anything, mock.Anything, 100).Return(nil, nil).Once()
repo.EXPECT().ListReferencedKeys(mock.Anything, "", 1000).
Return(nil, context.DeadlineExceeded).Once()
locker.EXPECT().AcquireSweepLock(mock.Anything).Return(func() {}, true, nil).Once()
sweeper := attachment.NewSweeper(repo, store, locker, time.Minute, time.Hour, slog.Default())
sweeper := attachment.NewSweeper(
repo, store, locker,
time.Minute, time.Hour, 24*time.Hour, time.Minute, slog.Default(),
)
stats, err := sweeper.SweepOnce(t.Context())
require.NoError(t, err)
require.Zero(t, stats.Recovered)
require.Zero(t, stats.RowsDeleted)
require.Contains(t, store.objects, row.ObjectKey)
_, err := sweeper.SweepOnce(t.Context())
require.ErrorIs(t, err, context.DeadlineExceeded)
require.Contains(t, store.objects, orphan, "fail-closed: no deletion when the key set is unknown")
}
@@ -269,3 +269,75 @@ func (_c *MockRepository_ListVisible_Call) RunAndReturn(run func(ctx context.Con
_c.Call.Return(run)
return _c
}
// Replay provides a mock function for the type MockRepository
func (_mock *MockRepository) Replay(ctx context.Context, draft domain.MessageDraft) (domain.Message, bool, error) {
ret := _mock.Called(ctx, draft)
if len(ret) == 0 {
panic("no return value specified for Replay")
}
var r0 domain.Message
var r1 bool
var r2 error
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.MessageDraft) (domain.Message, bool, error)); ok {
return returnFunc(ctx, draft)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, domain.MessageDraft) domain.Message); ok {
r0 = returnFunc(ctx, draft)
} else {
r0 = ret.Get(0).(domain.Message)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, domain.MessageDraft) bool); ok {
r1 = returnFunc(ctx, draft)
} else {
r1 = ret.Get(1).(bool)
}
if returnFunc, ok := ret.Get(2).(func(context.Context, domain.MessageDraft) error); ok {
r2 = returnFunc(ctx, draft)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// MockRepository_Replay_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Replay'
type MockRepository_Replay_Call struct {
*mock.Call
}
// Replay is a helper method to define mock.On call
// - ctx context.Context
// - draft domain.MessageDraft
func (_e *MockRepository_Expecter) Replay(ctx any, draft any) *MockRepository_Replay_Call {
return &MockRepository_Replay_Call{Call: _e.mock.On("Replay", ctx, draft)}
}
func (_c *MockRepository_Replay_Call) Run(run func(ctx context.Context, draft domain.MessageDraft)) *MockRepository_Replay_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 domain.MessageDraft
if args[1] != nil {
arg1 = args[1].(domain.MessageDraft)
}
run(
arg0,
arg1,
)
})
return _c
}
func (_c *MockRepository_Replay_Call) Return(message domain.Message, b bool, err error) *MockRepository_Replay_Call {
_c.Call.Return(message, b, err)
return _c
}
func (_c *MockRepository_Replay_Call) RunAndReturn(run func(ctx context.Context, draft domain.MessageDraft) (domain.Message, bool, error)) *MockRepository_Replay_Call {
_c.Call.Return(run)
return _c
}
+18 -1
View File
@@ -11,6 +11,11 @@ import (
type Repository interface {
IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error)
// Replay returns the stored message for a live Idempotency-Key. It must be
// checked before attachment-state validation: a message that was accepted
// while the attachment was queued keeps replaying as 201 even after the
// attachment later fails.
Replay(ctx context.Context, draft domain.MessageDraft) (domain.Message, bool, error)
Create(ctx context.Context, draft domain.MessageDraft) (domain.Message, error)
ListVisible(
ctx context.Context,
@@ -50,6 +55,15 @@ func (s *Service) Send(ctx context.Context, draft domain.MessageDraft) (domain.M
return domain.Message{}, fmt.Errorf("%w: chat is not visible", domain.ErrNotFound)
}
replayed, found, err := s.messages.Replay(ctx, draft)
if err != nil {
return domain.Message{}, fmt.Errorf("replay message: %w", err)
}
if found {
return replayed, nil
}
if len(draft.AttachmentIDs) > 0 {
if s.attachments == nil {
return domain.Message{}, fmt.Errorf("attachment checker is not configured: %w", domain.ErrUnavailable)
@@ -65,7 +79,10 @@ func (s *Service) Send(ctx context.Context, draft domain.MessageDraft) (domain.M
}
for _, state := range states {
if state.Status != domain.AttachmentReady {
switch state.Status {
case domain.AttachmentQueued, domain.AttachmentProcessing, domain.AttachmentReady:
// Accepted work is linkable by contract.
default:
return domain.Message{}, fmt.Errorf(
"%w: attachment %s is %s",
domain.ErrConflict, state.ID, state.Status,
+37
View File
@@ -6,6 +6,7 @@ import (
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"mtusi-messenger/internal/domain"
@@ -38,6 +39,8 @@ func (s *ServiceSuite) SetupTest() {
s.caller = uuid.New()
s.chatID = uuid.New()
s.now = time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)
s.messages.EXPECT().Replay(mock.Anything, mock.Anything).
Return(domain.Message{}, false, nil).Maybe()
}
func (s *ServiceSuite) draft() domain.MessageDraft {
@@ -167,6 +170,40 @@ func (s *ServiceSuite) TestSendAcceptsReadyAttachment() {
s.Equal([]uuid.UUID{attachmentID}, got.AttachmentIDs)
}
func (s *ServiceSuite) TestSendReplaysBeforeAttachmentValidation() {
draft := s.draft()
draft.AttachmentIDs = []uuid.UUID{uuid.New()}
stored := domain.Message{
ID: uuid.New(), ChatID: draft.ChatID, SenderID: draft.SenderID,
Text: draft.Text, AttachmentIDs: draft.AttachmentIDs, CreatedAt: s.now,
}
// The stored key is authoritative even if the attachment has since failed;
// ListStates must not be consulted on a replay.
s.messages.ExpectedCalls = nil
s.messages.EXPECT().IsChatMember(s.T().Context(), draft.ChatID, draft.SenderID).Return(true, nil).Once()
s.messages.EXPECT().Replay(s.T().Context(), draft).Return(stored, true, nil).Once()
got, err := s.service.Send(s.T().Context(), draft)
s.Require().NoError(err)
s.Equal(stored, got)
}
func (s *ServiceSuite) TestSendRejectsFailedAttachment() {
attachmentID := uuid.New()
draft := s.draft()
draft.AttachmentIDs = []uuid.UUID{attachmentID}
s.messages.EXPECT().IsChatMember(s.T().Context(), draft.ChatID, draft.SenderID).Return(true, nil).Once()
s.attachments.EXPECT().
ListStates(s.T().Context(), draft.ChatID, []uuid.UUID{attachmentID}).
Return([]domain.AttachmentState{{ID: attachmentID, Status: domain.AttachmentFailed}}, nil).
Once()
_, err := s.service.Send(s.T().Context(), draft)
s.Require().ErrorIs(err, domain.ErrConflict)
}
func (s *ServiceSuite) TestSendAllowsAttachmentOnlyMessage() {
attachmentID := uuid.New()
draft := s.draft()
@@ -1,215 +0,0 @@
package postgres
import (
"context"
"fmt"
"time"
"github.com/google/uuid"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/storage/postgres/internal/sqlcgen"
)
type AttachmentRepository struct {
store *Store
}
func (r *AttachmentRepository) IsChatMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
member, err := r.store.queries.IsChatMember(ctx, sqlcgen.IsChatMemberParams{
ChatID: chatID,
UserID: userID,
})
if err != nil {
return false, fmt.Errorf("check chat membership: %w", mapError(err))
}
return member, nil
}
func (r *AttachmentRepository) Create(ctx context.Context, draft domain.AttachmentDraft) (domain.Attachment, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
row, err := r.store.queries.CreateAttachment(ctx, sqlcgen.CreateAttachmentParams{
ID: draft.ID,
ChatID: draft.ChatID,
OwnerID: draft.OwnerID,
ObjectKey: draft.ObjectKey,
Filename: draft.Filename,
ContentType: draft.ContentType,
SizeBytes: draft.SizeBytes,
Sha256: draft.SHA256,
})
if err != nil {
return domain.Attachment{}, fmt.Errorf("create attachment: %w", mapError(err))
}
return mapAttachment(row), nil
}
func (r *AttachmentRepository) MarkReady(
ctx context.Context,
id uuid.UUID,
objectKey string,
) (domain.Attachment, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
row, err := r.store.queries.MarkAttachmentReady(ctx, sqlcgen.MarkAttachmentReadyParams{
ID: id,
ObjectKey: objectKey,
})
if err != nil {
return domain.Attachment{}, fmt.Errorf("mark attachment ready: %w", mapError(err))
}
return mapAttachment(row), nil
}
func (r *AttachmentRepository) DeletePending(ctx context.Context, id uuid.UUID) error {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
if err := r.store.queries.DeletePendingAttachment(ctx, id); err != nil {
return fmt.Errorf("delete pending attachment: %w", mapError(err))
}
return nil
}
func (r *AttachmentRepository) GetForMember(
ctx context.Context,
userID, id uuid.UUID,
) (domain.Attachment, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
row, err := r.store.queries.GetAttachmentForMember(ctx, sqlcgen.GetAttachmentForMemberParams{
ID: id,
UserID: userID,
})
if err != nil {
return domain.Attachment{}, fmt.Errorf("get attachment for member: %w", mapError(err))
}
return mapAttachment(row), nil
}
func (r *AttachmentRepository) GetForOwner(
ctx context.Context,
ownerID, id uuid.UUID,
) (domain.Attachment, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
row, err := r.store.queries.GetAttachmentForOwner(ctx, sqlcgen.GetAttachmentForOwnerParams{
ID: id,
OwnerID: ownerID,
})
if err != nil {
return domain.Attachment{}, fmt.Errorf("get attachment for owner: %w", mapError(err))
}
return mapAttachment(row), nil
}
func (r *AttachmentRepository) ListStates(
ctx context.Context,
chatID uuid.UUID,
ids []uuid.UUID,
) ([]domain.AttachmentState, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
rows, err := r.store.queries.ListAttachmentStatesForChat(ctx, sqlcgen.ListAttachmentStatesForChatParams{
ChatID: chatID,
Ids: ids,
})
if err != nil {
return nil, fmt.Errorf("list attachment states: %w", mapError(err))
}
states := make([]domain.AttachmentState, 0, len(rows))
for _, row := range rows {
states = append(states, domain.AttachmentState{
ID: row.ID,
Status: domain.AttachmentStatus(row.Status),
})
}
return states, nil
}
func (r *AttachmentRepository) ListStalePending(
ctx context.Context,
createdBefore time.Time,
limit int32,
) ([]domain.Attachment, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
rows, err := r.store.queries.ListStalePendingAttachments(ctx, sqlcgen.ListStalePendingAttachmentsParams{
CreatedBefore: createdBefore,
RowLimit: limit,
})
if err != nil {
return nil, fmt.Errorf("list stale attachments: %w", mapError(err))
}
return mapAll(rows, mapAttachment), nil
}
func (r *AttachmentRepository) ListKeys(
ctx context.Context,
limit, offset int32,
) ([]domain.AttachmentKey, error) {
ctx, cancel := r.store.timeout(ctx)
defer cancel()
rows, err := r.store.queries.ListAttachmentKeys(ctx, sqlcgen.ListAttachmentKeysParams{
RowLimit: limit,
RowOffset: offset,
})
if err != nil {
return nil, fmt.Errorf("list attachment keys: %w", mapError(err))
}
keys := make([]domain.AttachmentKey, 0, len(rows))
for _, row := range rows {
keys = append(keys, domain.AttachmentKey{
ID: row.ID,
ObjectKey: row.ObjectKey,
Status: domain.AttachmentStatus(row.Status),
CreatedAt: toUTCTime(row.CreatedAt),
})
}
return keys, nil
}
func mapAttachment(row sqlcgen.Attachment) domain.Attachment {
attachment := domain.Attachment{
ID: row.ID,
ChatID: row.ChatID,
OwnerID: row.OwnerID,
ObjectKey: row.ObjectKey,
Filename: row.Filename,
ContentType: row.ContentType,
SizeBytes: row.SizeBytes,
SHA256: row.Sha256,
Status: domain.AttachmentStatus(row.Status),
CreatedAt: toUTCTime(row.CreatedAt),
UpdatedAt: toUTCTime(row.UpdatedAt),
}
if row.ErrorCode.Valid {
code := row.ErrorCode.String
attachment.ErrorCode = &code
}
return attachment
}
@@ -1,92 +0,0 @@
//go:build integration
package postgres_test
import (
"strings"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"mtusi-messenger/internal/domain"
)
type AttachmentRepositorySuite struct {
BaseSuite
}
func TestAttachmentRepositorySuite(t *testing.T) {
suite.Run(t, new(AttachmentRepositorySuite))
}
func (s *AttachmentRepositorySuite) TestLifecycleAndMessageLinks() {
alice := s.createUser("alice")
bob := s.createUser("bob")
eve := s.createUser("eve")
chat := s.createChat(alice, bob)
id, err := uuid.NewV7()
s.Require().NoError(err)
objectKey := "attachments/" + chat.ID.String() + "/" + id.String()
digest := strings.Repeat("a", 64)
created, err := s.store.Attachments().Create(s.T().Context(), domain.AttachmentDraft{
ID: id,
ChatID: chat.ID,
OwnerID: alice.ID,
ObjectKey: objectKey,
Filename: "photo.png",
ContentType: "image/png",
SizeBytes: 123,
SHA256: digest,
})
s.Require().NoError(err)
s.Equal(domain.AttachmentPending, created.Status)
visible, err := s.store.Attachments().GetForMember(s.T().Context(), bob.ID, id)
s.Require().NoError(err)
s.Equal(alice.ID, visible.OwnerID)
_, err = s.store.Attachments().GetForMember(s.T().Context(), eve.ID, id)
s.Require().ErrorIs(err, domain.ErrNotFound)
ready, err := s.store.Attachments().MarkReady(s.T().Context(), id, objectKey)
s.Require().NoError(err)
s.Equal(domain.AttachmentReady, ready.Status)
states, err := s.store.Attachments().ListStates(
s.T().Context(), chat.ID, []uuid.UUID{id},
)
s.Require().NoError(err)
s.Require().Len(states, 1)
s.Equal(domain.AttachmentReady, states[0].Status)
key := uuid.NewString()
draft := domain.MessageDraft{
ChatID: chat.ID,
SenderID: alice.ID,
Text: "",
AttachmentIDs: []uuid.UUID{id},
Idempotency: domain.MessageIdempotency{
Key: key,
RequestHash: [32]byte{1},
},
}
message, err := s.store.Messages().Create(s.T().Context(), draft)
s.Require().NoError(err)
s.Equal([]uuid.UUID{id}, message.AttachmentIDs)
page, err := s.store.Messages().ListVisible(s.T().Context(), chat.ID, bob.ID, nil, 50)
s.Require().NoError(err)
s.Require().Len(page.Items, 1)
s.Equal([]uuid.UUID{id}, page.Items[0].AttachmentIDs)
s.Empty(page.Items[0].Text)
replayed, err := s.store.Messages().Create(s.T().Context(), draft)
s.Require().NoError(err)
s.Equal(message.ID, replayed.ID)
s.Equal([]uuid.UUID{id}, replayed.AttachmentIDs)
}
@@ -7,7 +7,6 @@ package sqlcgen
import (
"context"
"time"
"github.com/google/uuid"
)
@@ -28,228 +27,6 @@ func (q *Queries) AttachToMessage(ctx context.Context, arg AttachToMessageParams
return err
}
const createAttachment = `-- name: CreateAttachment :one
INSERT INTO attachments (
id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, status
)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8,
'pending'
)
RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at
`
type CreateAttachmentParams struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
ObjectKey string
Filename string
ContentType string
SizeBytes int64
Sha256 string
}
func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentParams) (Attachment, error) {
row := q.db.QueryRow(ctx, createAttachment,
arg.ID,
arg.ChatID,
arg.OwnerID,
arg.ObjectKey,
arg.Filename,
arg.ContentType,
arg.SizeBytes,
arg.Sha256,
)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.ObjectKey,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
const deletePendingAttachment = `-- name: DeletePendingAttachment :exec
DELETE FROM attachments
WHERE id = $1
AND status = 'pending'
`
func (q *Queries) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error {
_, err := q.db.Exec(ctx, deletePendingAttachment, id)
return err
}
const getAttachmentForMember = `-- name: GetAttachmentForMember :one
SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at
FROM attachments a
JOIN chat_members cm
ON cm.chat_id = a.chat_id
AND cm.user_id = $1
WHERE a.id = $2
`
type GetAttachmentForMemberParams struct {
UserID uuid.UUID
ID uuid.UUID
}
func (q *Queries) GetAttachmentForMember(ctx context.Context, arg GetAttachmentForMemberParams) (Attachment, error) {
row := q.db.QueryRow(ctx, getAttachmentForMember, arg.UserID, arg.ID)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.ObjectKey,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
const getAttachmentForOwner = `-- name: GetAttachmentForOwner :one
SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at
FROM attachments a
JOIN chat_members cm
ON cm.chat_id = a.chat_id
AND cm.user_id = a.owner_id
WHERE a.id = $1
AND a.owner_id = $2
`
type GetAttachmentForOwnerParams struct {
ID uuid.UUID
OwnerID uuid.UUID
}
func (q *Queries) GetAttachmentForOwner(ctx context.Context, arg GetAttachmentForOwnerParams) (Attachment, error) {
row := q.db.QueryRow(ctx, getAttachmentForOwner, arg.ID, arg.OwnerID)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.ObjectKey,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
const listAttachmentKeys = `-- name: ListAttachmentKeys :many
SELECT id, object_key, status, created_at
FROM attachments
ORDER BY id
LIMIT $2 OFFSET $1
`
type ListAttachmentKeysParams struct {
RowOffset int32
RowLimit int32
}
type ListAttachmentKeysRow struct {
ID uuid.UUID
ObjectKey string
Status string
CreatedAt time.Time
}
func (q *Queries) ListAttachmentKeys(ctx context.Context, arg ListAttachmentKeysParams) ([]ListAttachmentKeysRow, error) {
rows, err := q.db.Query(ctx, listAttachmentKeys, arg.RowOffset, arg.RowLimit)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListAttachmentKeysRow{}
for rows.Next() {
var i ListAttachmentKeysRow
if err := rows.Scan(
&i.ID,
&i.ObjectKey,
&i.Status,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listAttachmentStatesForChat = `-- name: ListAttachmentStatesForChat :many
SELECT id, status
FROM attachments
WHERE chat_id = $1
AND id = ANY($2::uuid[])
`
type ListAttachmentStatesForChatParams struct {
ChatID uuid.UUID
Ids []uuid.UUID
}
type ListAttachmentStatesForChatRow struct {
ID uuid.UUID
Status string
}
func (q *Queries) ListAttachmentStatesForChat(ctx context.Context, arg ListAttachmentStatesForChatParams) ([]ListAttachmentStatesForChatRow, error) {
rows, err := q.db.Query(ctx, listAttachmentStatesForChat, arg.ChatID, arg.Ids)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListAttachmentStatesForChatRow{}
for rows.Next() {
var i ListAttachmentStatesForChatRow
if err := rows.Scan(&i.ID, &i.Status); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listMessageAttachments = `-- name: ListMessageAttachments :many
SELECT message_id, attachment_id
FROM message_attachments
@@ -281,87 +58,3 @@ func (q *Queries) ListMessageAttachments(ctx context.Context, messageIds []uuid.
}
return items, nil
}
const listStalePendingAttachments = `-- name: ListStalePendingAttachments :many
SELECT id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at
FROM attachments
WHERE status = 'pending'
AND created_at < $1
ORDER BY created_at
LIMIT $2
`
type ListStalePendingAttachmentsParams struct {
CreatedBefore time.Time
RowLimit int32
}
func (q *Queries) ListStalePendingAttachments(ctx context.Context, arg ListStalePendingAttachmentsParams) ([]Attachment, error) {
rows, err := q.db.Query(ctx, listStalePendingAttachments, arg.CreatedBefore, arg.RowLimit)
if err != nil {
return nil, err
}
defer rows.Close()
items := []Attachment{}
for rows.Next() {
var i Attachment
if err := rows.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.ObjectKey,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const markAttachmentReady = `-- name: MarkAttachmentReady :one
UPDATE attachments
SET status = 'ready',
object_key = $1,
updated_at = now()
WHERE id = $2
AND status = 'pending'
RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at
`
type MarkAttachmentReadyParams struct {
ObjectKey string
ID uuid.UUID
}
func (q *Queries) MarkAttachmentReady(ctx context.Context, arg MarkAttachmentReadyParams) (Attachment, error) {
row := q.db.QueryRow(ctx, markAttachmentReady, arg.ObjectKey, arg.ID)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.ObjectKey,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
@@ -126,3 +126,32 @@ func (q *Queries) GetMessageIdempotency(ctx context.Context, arg GetMessageIdemp
err := row.Scan(&i.MessageID, &i.RequestHash)
return i, err
}
const lookupMessageIdempotency = `-- name: LookupMessageIdempotency :one
SELECT message_id, request_hash
FROM message_idempotency
WHERE actor_id = $1
AND chat_id = $2
AND key = $3
AND expires_at > now()
`
type LookupMessageIdempotencyParams struct {
ActorID uuid.UUID
ChatID uuid.UUID
Key string
}
type LookupMessageIdempotencyRow struct {
MessageID pgtype.UUID
RequestHash []byte
}
// Read-only replay lookup used before attachment-state validation. A live key
// returns the stored message; an expired key is ignored and may be claimed.
func (q *Queries) LookupMessageIdempotency(ctx context.Context, arg LookupMessageIdempotencyParams) (LookupMessageIdempotencyRow, error) {
row := q.db.QueryRow(ctx, lookupMessageIdempotency, arg.ActorID, arg.ChatID, arg.Key)
var i LookupMessageIdempotencyRow
err := row.Scan(&i.MessageID, &i.RequestHash)
return i, err
}
@@ -12,18 +12,62 @@ import (
)
type Attachment struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
ObjectKey string
Filename string
ContentType string
SizeBytes int64
Sha256 string
Status string
ErrorCode pgtype.Text
CreatedAt time.Time
UpdatedAt time.Time
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
ObjectKey string
Filename string
ContentType string
SizeBytes int64
Sha256 string
Status string
ErrorCode pgtype.Text
CreatedAt time.Time
UpdatedAt time.Time
PipelineVersion int32
StagingKey pgtype.Text
ReadyAt pgtype.Timestamptz
FailedAt pgtype.Timestamptz
}
type AttachmentJob struct {
JobID uuid.UUID
AttachmentID uuid.UUID
PipelineVersion int32
SourceKey string
Attempt int32
MaxAttempts int32
Status string
AvailableAt time.Time
LeaseOwner pgtype.Text
LeaseExpiresAt pgtype.Timestamptz
LastErrorCode pgtype.Text
ReplayCount int32
CreatedAt time.Time
UpdatedAt time.Time
CompletedAt pgtype.Timestamptz
}
type AttachmentJobEvent struct {
ID int64
JobID uuid.UUID
Event string
Attempt pgtype.Int4
Detail []byte
CreatedAt time.Time
}
type AttachmentVariant struct {
AttachmentID uuid.UUID
PipelineVersion int32
Name string
ContentType string
SizeBytes int64
Width int32
Height int32
Sha256 string
ObjectKey string
CreatedAt time.Time
}
type Chat struct {
@@ -62,6 +106,22 @@ type MessageIdempotency struct {
ExpiresAt time.Time
}
type OutboxMessage struct {
ID int64
JobID uuid.UUID
Attempt int32
Kind string
RoutingKey string
Payload []byte
AvailableAt time.Time
PublishedAt pgtype.Timestamptz
ClaimedBy pgtype.Text
ClaimedUntil pgtype.Timestamptz
PublishAttempts int32
LastError pgtype.Text
CreatedAt time.Time
}
type Session struct {
TokenHash []byte
UserID uuid.UUID
@@ -56,6 +56,61 @@ func (r *MessageRepository) Create(ctx context.Context, draft domain.MessageDraf
return created, nil
}
// Replay returns the stored message for a live Idempotency-Key without
// touching attachment state. The caller uses it before attachment validation so
// a replay stays 201 even if a linked attachment later failed.
func (r *MessageRepository) Replay(
ctx context.Context,
draft domain.MessageDraft,
) (domain.Message, bool, error) {
var (
replayed domain.Message
found bool
)
err := r.store.RunInTx(ctx, func(queries *sqlcgen.Queries) error {
stored, err := queries.LookupMessageIdempotency(ctx, sqlcgen.LookupMessageIdempotencyParams{
ActorID: draft.SenderID,
ChatID: draft.ChatID,
Key: draft.Idempotency.Key,
})
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return mapError(err)
}
if !bytes.Equal(stored.RequestHash, draft.Idempotency.RequestHash[:]) {
return domain.ErrIdempotency
}
if !stored.MessageID.Valid {
return errors.New("idempotency record without a message")
}
row, err := queries.GetMessageByID(ctx, uuid.UUID(stored.MessageID.Bytes))
if err != nil {
return mapError(err)
}
replayed = mapMessage(row.ID, row.ChatID, row.SenderID, row.Text, row.CreatedAt)
replayed.AttachmentIDs, err = listAttachmentIDs(ctx, queries, row.ID)
if err != nil {
return err
}
found = true
return nil
})
if err != nil {
return domain.Message{}, false, fmt.Errorf("replay idempotent message: %w", mapError(err))
}
return replayed, found, nil
}
// createIdempotent stores the message and its Idempotency-Key in one
// transaction: either both are committed or neither, so a replay never observes
// a claimed key whose message is missing. Claiming is a single statement, so
@@ -0,0 +1,150 @@
-- +goose Up
-- Lab5 async attachment pipeline: durable intent (jobs + transactional outbox),
-- per-version results (variants), and operator audit (events).
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (
status IN ('pending', 'queued', 'processing', 'ready', 'failed')
) NOT VALID;
ALTER TABLE attachments VALIDATE CONSTRAINT attachments_status_check;
ALTER TABLE attachments DROP CONSTRAINT attachments_error_consistency;
ALTER TABLE attachments ADD CONSTRAINT attachments_error_consistency CHECK (
(status IN ('ready', 'queued', 'processing') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
status = 'pending'
) NOT VALID;
ALTER TABLE attachments VALIDATE CONSTRAINT attachments_error_consistency;
ALTER TABLE attachments
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1
CHECK (pipeline_version >= 1),
ADD COLUMN staging_key text,
ADD COLUMN ready_at timestamptz,
ADD COLUMN failed_at timestamptz;
CREATE UNIQUE INDEX attachments_staging_key_key
ON attachments (staging_key)
WHERE staging_key IS NOT NULL;
CREATE INDEX attachments_pipeline_status_idx
ON attachments (status, pipeline_version);
-- One logical job per (attachment, pipeline version). job_id is stable across
-- redelivery; attempt is the managed attempt number (1-based).
CREATE TABLE attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
source_key text NOT NULL,
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 100),
status text NOT NULL CHECK (
status IN ('queued', 'processing', 'ready', 'failed', 'superseded')
),
available_at timestamptz NOT NULL DEFAULT now(),
lease_owner text,
lease_expires_at timestamptz,
last_error_code text,
replay_count integer NOT NULL DEFAULT 0,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
completed_at timestamptz,
CONSTRAINT attachment_jobs_attachment_version_key UNIQUE (attachment_id, pipeline_version),
CONSTRAINT attachment_jobs_lease_consistency CHECK (
(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) OR
(status <> 'processing')
)
);
CREATE INDEX attachment_jobs_due_idx
ON attachment_jobs (status, available_at);
CREATE INDEX attachment_jobs_lease_idx
ON attachment_jobs (status, lease_expires_at);
-- Delivery ledger. available_at is the only retry scheduler; RabbitMQ is the
-- durable transport. No UNIQUE(job_id, attempt): a graceful shutdown may
-- re-publish the same attempt, and duplicates are harmless (worker CAS).
CREATE TABLE outbox_messages (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
attempt integer NOT NULL CHECK (attempt >= 1),
kind text NOT NULL DEFAULT 'publish',
routing_key text NOT NULL,
payload jsonb NOT NULL,
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
claimed_by text,
claimed_until timestamptz,
publish_attempts integer NOT NULL DEFAULT 0,
last_error text,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX outbox_due_idx
ON outbox_messages (available_at, id)
WHERE published_at IS NULL;
CREATE INDEX outbox_claim_expiry_idx
ON outbox_messages (claimed_until)
WHERE published_at IS NULL AND claimed_by IS NOT NULL;
-- Effect key (attachment_id, pipeline_version, name). The primary key is the
-- idempotency boundary for redelivery and versioned pipelines.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
content_type text NOT NULL CHECK (
content_type IN ('image/jpeg', 'image/png', 'image/webp')
),
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
width integer NOT NULL CHECK (width > 0),
height integer NOT NULL CHECK (height > 0),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, name)
);
CREATE TABLE attachment_job_events (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
event text NOT NULL,
attempt integer,
detail jsonb,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX attachment_job_events_job_idx
ON attachment_job_events (job_id, created_at);
-- +goose Down
DROP TABLE attachment_job_events;
DROP TABLE attachment_variants;
DROP TABLE outbox_messages;
DROP TABLE attachment_jobs;
DROP INDEX attachments_pipeline_status_idx;
DROP INDEX attachments_staging_key_key;
ALTER TABLE attachments
DROP COLUMN failed_at,
DROP COLUMN ready_at,
DROP COLUMN staging_key,
DROP COLUMN pipeline_version;
ALTER TABLE attachments DROP CONSTRAINT attachments_error_consistency;
ALTER TABLE attachments ADD CONSTRAINT attachments_error_consistency CHECK (
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
status = 'pending'
) NOT VALID;
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (
status IN ('pending', 'ready', 'failed')
) NOT VALID;
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,552 @@
//go:build integration
package postgres_test
import (
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"mtusi-messenger/internal/domain"
)
type PipelineSuite struct {
BaseSuite
}
func TestPipelineSuite(t *testing.T) {
suite.Run(t, new(PipelineSuite))
}
func (s *PipelineSuite) attachmentDraft(chat domain.Chat, owner domain.User, version int) (domain.AttachmentDraft, domain.Job) {
s.T().Helper()
id, err := uuid.NewV7()
s.Require().NoError(err)
jobID, err := uuid.NewV7()
s.Require().NoError(err)
draft := domain.AttachmentDraft{
ID: id,
ChatID: chat.ID,
OwnerID: owner.ID,
ObjectKey: domain.OriginalObjectKey(chat.ID, id),
StagingKey: domain.StagingUploadKey(chat.ID, id),
Filename: "photo.png",
ContentType: "image/png",
SizeBytes: 10,
SHA256: strings.Repeat("a", 64),
Status: domain.AttachmentQueued,
PipelineVersion: version,
}
job := domain.Job{
ID: jobID,
AttachmentID: id,
PipelineVersion: version,
SourceKey: draft.StagingKey,
Attempt: 1,
MaxAttempts: 5,
CreatedAt: time.Now().UTC(),
}
return draft, job
}
func (s *PipelineSuite) TestCreateClaimCompleteAndIdempotentRedelivery() {
alice := s.createUser("alice")
bob := s.createUser("bob")
chat := s.createChat(alice, bob)
draft, job := s.attachmentDraft(chat, alice, 1)
repo := s.store.Pipeline()
created, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
s.Equal(domain.AttachmentQueued, created.Status)
s.Equal(1, created.PipelineVersion)
message := domain.NewJobMessage(job)
claim, err := repo.ClaimJob(s.T().Context(), message, "lease-1", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
s.Equal(domain.AttachmentProcessing, claim.Attachment.Status)
duplicate, err := repo.ClaimJob(s.T().Context(), message, "lease-2", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimBusyLease, duplicate.Outcome)
variants := []domain.Variant{
{
Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11,
Width: 256, Height: 160, SHA256: strings.Repeat("b", 64),
ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail),
},
}
ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants)
s.Require().NoError(err)
s.True(ok)
ready, err := repo.GetForMember(s.T().Context(), bob.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentReady, ready.Status)
stored, err := repo.ListVariants(s.T().Context(), draft.ID, 1)
s.Require().NoError(err)
s.Require().Len(stored, 1)
s.Equal(domain.VariantThumbnail, stored[0].Name)
// A redelivery after ready is a no-op, even with the same attempt.
again, err := repo.ClaimJob(s.T().Context(), message, "lease-3", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAlreadyReady, again.Outcome)
}
func (s *PipelineSuite) TestLeaseExpiryIsBoundedByMaxAttempts() {
alice := s.createUser("alice")
chat := s.createChat(alice)
draft, job := s.attachmentDraft(chat, alice, 1)
job.MaxAttempts = 1
repo := s.store.Pipeline()
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease", time.Millisecond)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
time.Sleep(10 * time.Millisecond)
reaped, err := repo.ReapExpiredLeases(s.T().Context(), 10)
s.Require().NoError(err)
s.Equal(1, reaped)
dead, err := repo.GetJob(s.T().Context(), job.ID)
s.Require().NoError(err)
s.Equal(domain.JobFailed, dead.Status)
s.NotNil(dead.LastErrorCode)
s.Equal("retry_exhausted", *dead.LastErrorCode)
}
func (s *PipelineSuite) TestAdmissionCountsPendingPresignAndIsAtomic() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, _ := s.attachmentDraft(chat, alice, 1)
draft.ObjectKey = "staging/presign/" + chat.ID.String() + "/" + draft.ID.String()
draft.StagingKey = draft.ObjectKey
draft.Status = domain.AttachmentPending
limits := domain.AdmissionLimits{PerUser: 1, Total: 10}
_, err := repo.CreatePresign(s.T().Context(), draft, limits)
s.Require().NoError(err)
second, _ := s.attachmentDraft(chat, alice, 1)
second.ObjectKey = "staging/presign/" + chat.ID.String() + "/" + second.ID.String()
second.StagingKey = second.ObjectKey
second.Status = domain.AttachmentPending
_, err = repo.CreatePresign(s.T().Context(), second, limits)
s.Require().ErrorIs(err, domain.ErrRateLimited)
}
// A queued upload whose worker never ran still has its verified bytes only in
// staging. Reprocessing it must use staging as the source; pointing at the
// not-yet-written final original left the attachment queued forever after the
// v1 job was superseded (v2 then failed with source_missing).
func (s *PipelineSuite) TestReprocessQueuedUsesStagingSource() {
alice := s.createUser("alice")
bob := s.createUser("bob")
chat := s.createChat(alice, bob)
draft, job := s.attachmentDraft(chat, alice, 1)
repo := s.store.Pipeline()
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
created, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
s.Equal(2, created.PipelineVersion)
s.Equal(draft.StagingKey, created.SourceKey)
}
// Once a pipeline has published the final original, the staging copy may be
// past its retention window; a reprocess of a ready attachment must source
// from the final object that actually exists.
func (s *PipelineSuite) TestReprocessReadyUsesObjectSource() {
alice := s.createUser("alice")
chat := s.createChat(alice)
draft, job := s.attachmentDraft(chat, alice, 1)
repo := s.store.Pipeline()
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
message := domain.NewJobMessage(job)
claim, err := repo.ClaimJob(s.T().Context(), message, "lease-1", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
variants := []domain.Variant{
{
Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11,
Width: 256, Height: 160, SHA256: strings.Repeat("b", 64),
ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail),
},
}
ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants)
s.Require().NoError(err)
s.True(ok)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
created, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
s.Equal(2, created.PipelineVersion)
s.Equal(draft.ObjectKey, created.SourceKey)
}
// A reprocess failure before any version was published must settle the
// attachment as failed instead of leaving it queued with no active jobs.
func (s *PipelineSuite) TestReprocessFailureSettlesUnpublishedAttachment() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image")
s.Require().NoError(err)
s.True(failed)
current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentFailed, current.Status)
s.Require().NotNil(current.ErrorCode)
s.Equal("invalid_image", *current.ErrorCode)
}
// A reprocess failure while an older ready version is serving must keep that
// version ready and downloadable; only the new job fails.
func (s *PipelineSuite) TestReprocessFailureKeepsReadyServingVersion() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-1", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
variants := []domain.Variant{
{
Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11,
Width: 256, Height: 160, SHA256: strings.Repeat("b", 64),
ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail),
},
}
ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants)
s.Require().NoError(err)
s.True(ok)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
claim, err = repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image")
s.Require().NoError(err)
s.True(failed)
current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentReady, current.Status)
s.Equal(1, current.PipelineVersion)
s.Nil(current.ErrorCode)
}
// Replaying a failed newer version must clear the stale terminal projection
// for an attachment that never published any version.
func (s *PipelineSuite) TestReplayNewerFailedVersionRequeuesUnpublishedAttachment() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image")
s.Require().NoError(err)
s.True(failed)
current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentFailed, current.Status)
replayed, err := repo.ReplayJob(s.T().Context(), reprocess.ID, 2)
s.Require().NoError(err)
s.Equal(2, replayed.Attempt)
current, err = repo.GetForOwner(s.T().Context(), alice.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentQueued, current.Status)
s.Nil(current.ErrorCode)
claim, err = repo.ClaimJob(s.T().Context(), domain.NewJobMessage(replayed), "lease-3", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
variants := []domain.Variant{
{
Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11,
Width: 256, Height: 160, SHA256: strings.Repeat("c", 64),
ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 2, domain.VariantThumbnail),
},
}
ok, err := repo.CompleteJob(s.T().Context(), replayed.ID, replayed.Attempt, "lease-3", variants)
s.Require().NoError(err)
s.True(ok)
current, err = repo.GetForOwner(s.T().Context(), alice.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentReady, current.Status)
s.Equal(2, current.PipelineVersion)
}
// Replaying a failed newer version must not disturb a published ready
// serving version; only the new job returns to queued.
func (s *PipelineSuite) TestReplayNewerFailedVersionKeepsReadyServingVersion() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-1", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
variants := []domain.Variant{
{
Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11,
Width: 256, Height: 160, SHA256: strings.Repeat("b", 64),
ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 1, domain.VariantThumbnail),
},
}
ok, err := repo.CompleteJob(s.T().Context(), job.ID, 1, "lease-1", variants)
s.Require().NoError(err)
s.True(ok)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
claim, err = repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
failed, err := repo.FailJob(s.T().Context(), reprocess.ID, 1, "lease-2", "invalid_image")
s.Require().NoError(err)
s.True(failed)
replayed, err := repo.ReplayJob(s.T().Context(), reprocess.ID, 1)
s.Require().NoError(err)
s.Equal(2, replayed.Attempt)
current, err := repo.GetForOwner(s.T().Context(), alice.ID, draft.ID)
s.Require().NoError(err)
s.Equal(domain.AttachmentReady, current.Status)
s.Equal(1, current.PipelineVersion)
s.Nil(current.ErrorCode)
}
// Replaying an older failed version is refused while a newer job exists,
// even before that newer version becomes the published serving version.
func (s *PipelineSuite) TestReplayOlderFailedVersionRefusedWhenNewerJobExists() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-1", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
failed, err := repo.FailJob(s.T().Context(), job.ID, 1, "lease-1", "invalid_image")
s.Require().NoError(err)
s.True(failed)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
_, err = repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
_, err = repo.ReplayJob(s.T().Context(), job.ID, 1)
s.Require().ErrorIs(err, domain.ErrConflict)
var v1Status string
s.Require().NoError(s.pool.QueryRow(s.T().Context(),
`SELECT status FROM attachment_jobs WHERE job_id = $1`, job.ID).Scan(&v1Status))
s.Equal(string(domain.JobFailed), v1Status)
}
// A queued job for an older version is retired on claim instead of being
// acknowledged and republished by reconciliation forever.
func (s *PipelineSuite) TestClaimRetiresVersionStaleQueuedJob() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
reprocessID, err := uuid.NewV7()
s.Require().NoError(err)
reprocess, err := repo.Reprocess(s.T().Context(), draft.ID, 2, 5, reprocessID)
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(reprocess), "lease-2", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
variants := []domain.Variant{
{
Name: domain.VariantThumbnail, ContentType: "image/jpeg", SizeBytes: 11,
Width: 256, Height: 160, SHA256: strings.Repeat("d", 64),
ObjectKey: domain.VariantObjectKey(chat.ID, draft.ID, 2, domain.VariantThumbnail),
},
}
ok, err := repo.CompleteJob(s.T().Context(), reprocess.ID, 1, "lease-2", variants)
s.Require().NoError(err)
s.True(ok)
// Simulate a legacy/racing replay that left the old version queued.
_, err = s.pool.Exec(s.T().Context(), `
UPDATE attachment_jobs
SET status = 'queued', attempt = 1, last_error_code = NULL
WHERE job_id = $1`, job.ID)
s.Require().NoError(err)
stale, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease-3", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimVersionStale, stale.Outcome)
var status string
s.Require().NoError(s.pool.QueryRow(s.T().Context(),
`SELECT status FROM attachment_jobs WHERE job_id = $1`, job.ID).Scan(&status))
s.Equal(string(domain.JobSuperseded), status)
var supersededEvents int
s.Require().NoError(s.pool.QueryRow(s.T().Context(), `
SELECT count(*) FROM attachment_job_events
WHERE job_id = $1 AND event = $2`,
job.ID, string(domain.EventSuperseded)).Scan(&supersededEvents))
s.Equal(2, supersededEvents)
}
func (s *PipelineSuite) TestReconcileQueuedJobsRepublishesLostIntent() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
claimed, err := repo.ClaimOutbox(s.T().Context(), "relay", time.Minute, 10)
s.Require().NoError(err)
s.Require().Len(claimed, 1)
s.Require().NoError(repo.MarkPublished(s.T().Context(), []int64{claimed[0].ID}))
reconciled, err := repo.ReconcileQueuedJobs(s.T().Context(), 0, 10)
s.Require().NoError(err)
s.Equal(1, reconciled)
pending, err := repo.ClaimOutbox(s.T().Context(), "relay", time.Minute, 10)
s.Require().NoError(err)
s.Require().Len(pending, 1)
s.Equal("reconcile", pending[0].Kind)
s.Equal(job.ID, pending[0].JobID)
}
func (s *PipelineSuite) TestFailedStagingStaysReferencedUntilRetention() {
alice := s.createUser("alice")
chat := s.createChat(alice)
repo := s.store.Pipeline()
draft, job := s.attachmentDraft(chat, alice, 1)
_, err := repo.CreateQueued(s.T().Context(), draft, job, domain.AdmissionLimits{})
s.Require().NoError(err)
claim, err := repo.ClaimJob(s.T().Context(), domain.NewJobMessage(job), "lease", time.Minute)
s.Require().NoError(err)
s.Equal(domain.ClaimAcquired, claim.Outcome)
ok, err := repo.FailJob(s.T().Context(), job.ID, 1, "lease", "invalid_image")
s.Require().NoError(err)
s.True(ok)
keys, err := repo.ListReferencedKeys(s.T().Context(), "", 100)
s.Require().NoError(err)
s.Contains(keys, draft.StagingKey)
}
@@ -1,56 +1,3 @@
-- name: CreateAttachment :one
INSERT INTO attachments (
id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, status
)
VALUES (
sqlc.arg(id),
sqlc.arg(chat_id),
sqlc.arg(owner_id),
sqlc.arg(object_key),
sqlc.arg(filename),
sqlc.arg(content_type),
sqlc.arg(size_bytes),
sqlc.arg(sha256),
'pending'
)
RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at;
-- name: MarkAttachmentReady :one
UPDATE attachments
SET status = 'ready',
object_key = sqlc.arg(object_key),
updated_at = now()
WHERE id = sqlc.arg(id)
AND status = 'pending'
RETURNING id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at;
-- name: GetAttachmentForMember :one
SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at
FROM attachments a
JOIN chat_members cm
ON cm.chat_id = a.chat_id
AND cm.user_id = sqlc.arg(user_id)
WHERE a.id = sqlc.arg(id);
-- name: GetAttachmentForOwner :one
SELECT a.id, a.chat_id, a.owner_id, a.object_key, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at
FROM attachments a
JOIN chat_members cm
ON cm.chat_id = a.chat_id
AND cm.user_id = a.owner_id
WHERE a.id = sqlc.arg(id)
AND a.owner_id = sqlc.arg(owner_id);
-- name: ListAttachmentStatesForChat :many
SELECT id, status
FROM attachments
WHERE chat_id = sqlc.arg(chat_id)
AND id = ANY(sqlc.arg(ids)::uuid[]);
-- name: AttachToMessage :exec
INSERT INTO message_attachments (message_id, attachment_id, position)
VALUES (sqlc.arg(message_id), sqlc.arg(attachment_id), sqlc.arg(position));
@@ -60,23 +7,3 @@ SELECT message_id, attachment_id
FROM message_attachments
WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
ORDER BY message_id, position;
-- name: DeletePendingAttachment :exec
DELETE FROM attachments
WHERE id = sqlc.arg(id)
AND status = 'pending';
-- name: ListStalePendingAttachments :many
SELECT id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at
FROM attachments
WHERE status = 'pending'
AND created_at < sqlc.arg(created_before)
ORDER BY created_at
LIMIT sqlc.arg(row_limit);
-- name: ListAttachmentKeys :many
SELECT id, object_key, status, created_at
FROM attachments
ORDER BY id
LIMIT sqlc.arg(row_limit) OFFSET sqlc.arg(row_offset);
@@ -38,3 +38,13 @@ WHERE actor_id = sqlc.arg(actor_id)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE id = sqlc.arg(id);
-- Read-only replay lookup used before attachment-state validation. A live key
-- returns the stored message; an expired key is ignored and may be claimed.
-- name: LookupMessageIdempotency :one
SELECT message_id, request_hash
FROM message_idempotency
WHERE actor_id = sqlc.arg(actor_id)
AND chat_id = sqlc.arg(chat_id)
AND key = sqlc.arg(key)
AND expires_at > now();
+7 -7
View File
@@ -17,11 +17,11 @@ type Store struct {
queryTimeout time.Duration
idempotencyTTL time.Duration
users *UserRepository
chats *ChatRepository
messages *MessageRepository
sessions *SessionRepository
attachments *AttachmentRepository
users *UserRepository
chats *ChatRepository
messages *MessageRepository
sessions *SessionRepository
pipeline *PipelineRepository
}
func New(ctx context.Context, cfg config.Postgres, idempotencyTTL time.Duration) (*Store, error) {
@@ -66,7 +66,7 @@ func New(ctx context.Context, cfg config.Postgres, idempotencyTTL time.Duration)
store.chats = &ChatRepository{store: store}
store.messages = &MessageRepository{store: store}
store.sessions = &SessionRepository{store: store}
store.attachments = &AttachmentRepository{store: store}
store.pipeline = &PipelineRepository{store: store}
return store, nil
}
@@ -79,7 +79,7 @@ func (s *Store) Messages() *MessageRepository { return s.messages }
func (s *Store) Sessions() *SessionRepository { return s.sessions }
func (s *Store) Attachments() *AttachmentRepository { return s.attachments }
func (s *Store) Pipeline() *PipelineRepository { return s.pipeline }
func (s *Store) Ping(ctx context.Context) error {
if err := s.pool.Ping(ctx); err != nil {
+42 -6
View File
@@ -56,11 +56,14 @@ func (h *Handler) DownloadAttachment(
ctx context.Context,
params oas.DownloadAttachmentParams,
) (oas.DownloadAttachmentRes, error) {
if variant, ok := params.Variant.Get(); ok && variant != oas.DownloadAttachmentVariantOriginal {
return nil, fmt.Errorf("%w: variant %q is not available", domain.ErrNotFound, variant)
variant := domain.VariantOriginal
if requested, ok := params.Variant.Get(); ok {
variant = domain.VariantName(requested)
}
attachment, body, err := h.attachments.Open(ctx, caller(ctx), params.AttachmentID)
attachment, metadata, body, err := h.attachments.OpenVariant(
ctx, caller(ctx), params.AttachmentID, variant,
)
if err != nil {
return nil, err
}
@@ -73,7 +76,7 @@ func (h *Handler) DownloadAttachment(
requestID := oas.NewOptString(requestIDFromContext(ctx))
reader := closeOnEOF{ReadCloser: body}
switch attachment.ContentType {
switch metadata.ContentType {
case "image/png":
return &oas.DownloadAttachmentOKImagePNGHeaders{
XRequestID: requestID,
@@ -84,6 +87,11 @@ func (h *Handler) DownloadAttachment(
XRequestID: requestID,
Response: oas.DownloadAttachmentOKImageJpeg{Data: reader},
}, nil
case "image/webp":
return &oas.DownloadAttachmentOKImageWEBPHeaders{
XRequestID: requestID,
Response: oas.DownloadAttachmentOKImageWEBP{Data: reader},
}, nil
default:
_ = body.Close()
@@ -110,11 +118,11 @@ func attachmentResponse(attachment domain.Attachment) oas.Attachment {
Filename: attachment.Filename,
SizeBytes: int(attachment.SizeBytes),
SHA256: attachment.SHA256,
Status: oas.AttachmentStatusReady,
Variants: []oas.Variant{},
Status: attachmentStatus(attachment.Status),
ErrorCode: oas.NilString{Null: true},
CreatedAt: attachment.CreatedAt,
UpdatedAt: attachment.UpdatedAt,
Variants: make([]oas.Variant, 0, len(attachment.Variants)),
}
switch attachment.ContentType {
@@ -130,9 +138,37 @@ func attachmentResponse(attachment domain.Attachment) oas.Attachment {
response.ErrorCode = oas.NilString{Value: *attachment.ErrorCode}
}
for _, variant := range attachment.Variants {
response.Variants = append(response.Variants, oas.Variant{
Name: oas.VariantName(variant.Name),
ContentType: oas.VariantContentType(variant.ContentType),
SizeBytes: int(variant.SizeBytes),
Width: variant.Width,
Height: variant.Height,
SHA256: variant.SHA256,
})
}
return response
}
func attachmentStatus(status domain.AttachmentStatus) oas.AttachmentStatus {
switch status {
case domain.AttachmentQueued:
return oas.AttachmentStatusQueued
case domain.AttachmentProcessing:
return oas.AttachmentStatusProcessing
case domain.AttachmentReady:
return oas.AttachmentStatusReady
case domain.AttachmentFailed:
return oas.AttachmentStatusFailed
default:
// Pending is internal and never reaches the transport; map it to the
// closest public state rather than inventing a new enum value.
return oas.AttachmentStatusQueued
}
}
type closeOnEOF struct {
io.ReadCloser
}
+2
View File
@@ -60,6 +60,8 @@ func statusForDomainError(err error) int {
errors.Is(err, domain.ErrUnknownMember),
errors.Is(err, domain.ErrInvalidCursor):
return http.StatusBadRequest
case errors.Is(err, domain.ErrRateLimited):
return http.StatusTooManyRequests
case errors.Is(err, domain.ErrTooLarge):
return http.StatusRequestEntityTooLarge
case errors.Is(err, domain.ErrUnsupportedMedia):
+1 -1
View File
@@ -176,7 +176,7 @@ func (e *Extensions) complete(w http.ResponseWriter, r *http.Request, userID uui
return err
}
return writeJSON(w, http.StatusOK, attachmentResponse(attachment))
return writeJSON(w, http.StatusAccepted, attachmentResponse(attachment))
}
func decodeJSON(r *http.Request, target any) error {
+5 -1
View File
@@ -56,7 +56,11 @@ type SessionService interface {
type AttachmentService interface {
Upload(ctx context.Context, input attachmentService.UploadInput) (domain.Attachment, error)
Get(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, error)
Open(ctx context.Context, callerID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error)
OpenVariant(
ctx context.Context,
callerID, id uuid.UUID,
name domain.VariantName,
) (domain.Attachment, domain.Variant, io.ReadCloser, error)
Presign(ctx context.Context, request domain.PresignRequest) (domain.PresignResult, error)
Complete(ctx context.Context, ownerID, id uuid.UUID, sha256 string) (domain.Attachment, error)
}
@@ -357,8 +357,14 @@ func (s *AttachmentStatus) Decode(d *jx.Decoder) error {
}
// Try to use constant string.
switch AttachmentStatus(v) {
case AttachmentStatusQueued:
*s = AttachmentStatusQueued
case AttachmentStatusProcessing:
*s = AttachmentStatusProcessing
case AttachmentStatusReady:
*s = AttachmentStatusReady
case AttachmentStatusFailed:
*s = AttachmentStatusFailed
default:
*s = AttachmentStatus(v)
}
@@ -4592,7 +4592,7 @@ func encodeUploadAttachmentResponse(response UploadAttachmentRes, w http.Respons
}
}
}
w.WriteHeader(201)
w.WriteHeader(202)
e := new(jx.Encoder)
response.Response.Encode(e)
@@ -230,21 +230,33 @@ func (*AttachmentHeaders) uploadAttachmentRes() {}
type AttachmentStatus string
const (
AttachmentStatusReady AttachmentStatus = "ready"
AttachmentStatusQueued AttachmentStatus = "queued"
AttachmentStatusProcessing AttachmentStatus = "processing"
AttachmentStatusReady AttachmentStatus = "ready"
AttachmentStatusFailed AttachmentStatus = "failed"
)
// AllValues returns all AttachmentStatus values.
func (AttachmentStatus) AllValues() []AttachmentStatus {
return []AttachmentStatus{
AttachmentStatusQueued,
AttachmentStatusProcessing,
AttachmentStatusReady,
AttachmentStatusFailed,
}
}
// MarshalText implements encoding.TextMarshaler.
func (s AttachmentStatus) MarshalText() ([]byte, error) {
switch s {
case AttachmentStatusQueued:
return []byte(s), nil
case AttachmentStatusProcessing:
return []byte(s), nil
case AttachmentStatusReady:
return []byte(s), nil
case AttachmentStatusFailed:
return []byte(s), nil
default:
return nil, errors.Errorf("invalid value: %q", s)
}
@@ -253,9 +265,18 @@ func (s AttachmentStatus) MarshalText() ([]byte, error) {
// UnmarshalText implements encoding.TextUnmarshaler.
func (s *AttachmentStatus) UnmarshalText(data []byte) error {
switch AttachmentStatus(data) {
case AttachmentStatusQueued:
*s = AttachmentStatusQueued
return nil
case AttachmentStatusProcessing:
*s = AttachmentStatusProcessing
return nil
case AttachmentStatusReady:
*s = AttachmentStatusReady
return nil
case AttachmentStatusFailed:
*s = AttachmentStatusFailed
return nil
default:
return errors.Errorf("invalid value: %q", data)
}
@@ -818,13 +839,19 @@ func (*DownloadAttachmentServiceUnavailable) downloadAttachmentRes() {}
type DownloadAttachmentVariant string
const (
DownloadAttachmentVariantOriginal DownloadAttachmentVariant = "original"
DownloadAttachmentVariantOriginal DownloadAttachmentVariant = "original"
DownloadAttachmentVariantThumbnail DownloadAttachmentVariant = "thumbnail"
DownloadAttachmentVariantCrop DownloadAttachmentVariant = "crop"
DownloadAttachmentVariantOptimized DownloadAttachmentVariant = "optimized"
)
// AllValues returns all DownloadAttachmentVariant values.
func (DownloadAttachmentVariant) AllValues() []DownloadAttachmentVariant {
return []DownloadAttachmentVariant{
DownloadAttachmentVariantOriginal,
DownloadAttachmentVariantThumbnail,
DownloadAttachmentVariantCrop,
DownloadAttachmentVariantOptimized,
}
}
@@ -833,6 +860,12 @@ func (s DownloadAttachmentVariant) MarshalText() ([]byte, error) {
switch s {
case DownloadAttachmentVariantOriginal:
return []byte(s), nil
case DownloadAttachmentVariantThumbnail:
return []byte(s), nil
case DownloadAttachmentVariantCrop:
return []byte(s), nil
case DownloadAttachmentVariantOptimized:
return []byte(s), nil
default:
return nil, errors.Errorf("invalid value: %q", s)
}
@@ -844,6 +877,15 @@ func (s *DownloadAttachmentVariant) UnmarshalText(data []byte) error {
case DownloadAttachmentVariantOriginal:
*s = DownloadAttachmentVariantOriginal
return nil
case DownloadAttachmentVariantThumbnail:
*s = DownloadAttachmentVariantThumbnail
return nil
case DownloadAttachmentVariantCrop:
*s = DownloadAttachmentVariantCrop
return nil
case DownloadAttachmentVariantOptimized:
*s = DownloadAttachmentVariantOptimized
return nil
default:
return errors.Errorf("invalid value: %q", data)
}
@@ -234,8 +234,14 @@ func (s *AttachmentHeaders) Validate() error {
func (s AttachmentStatus) Validate() error {
switch s {
case "queued":
return nil
case "processing":
return nil
case "ready":
return nil
case "failed":
return nil
default:
return errors.Errorf("invalid value: %v", s)
}
@@ -1103,6 +1109,12 @@ func (s DownloadAttachmentVariant) Validate() error {
switch s {
case "original":
return nil
case "thumbnail":
return nil
case "crop":
return nil
case "optimized":
return nil
default:
return errors.Errorf("invalid value: %v", s)
}
@@ -200,54 +200,61 @@ func (_c *MockAttachmentService_Get_Call) RunAndReturn(run func(ctx context.Cont
return _c
}
// Open provides a mock function for the type MockAttachmentService
func (_mock *MockAttachmentService) Open(ctx context.Context, callerID uuid.UUID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error) {
ret := _mock.Called(ctx, callerID, id)
// OpenVariant provides a mock function for the type MockAttachmentService
func (_mock *MockAttachmentService) OpenVariant(ctx context.Context, callerID uuid.UUID, id uuid.UUID, name domain.VariantName) (domain.Attachment, domain.Variant, io.ReadCloser, error) {
ret := _mock.Called(ctx, callerID, id, name)
if len(ret) == 0 {
panic("no return value specified for Open")
panic("no return value specified for OpenVariant")
}
var r0 domain.Attachment
var r1 io.ReadCloser
var r2 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID) (domain.Attachment, io.ReadCloser, error)); ok {
return returnFunc(ctx, callerID, id)
var r1 domain.Variant
var r2 io.ReadCloser
var r3 error
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) (domain.Attachment, domain.Variant, io.ReadCloser, error)); ok {
return returnFunc(ctx, callerID, id, name)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID) domain.Attachment); ok {
r0 = returnFunc(ctx, callerID, id)
if returnFunc, ok := ret.Get(0).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) domain.Attachment); ok {
r0 = returnFunc(ctx, callerID, id, name)
} else {
r0 = ret.Get(0).(domain.Attachment)
}
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, uuid.UUID) io.ReadCloser); ok {
r1 = returnFunc(ctx, callerID, id)
if returnFunc, ok := ret.Get(1).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) domain.Variant); ok {
r1 = returnFunc(ctx, callerID, id, name)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(io.ReadCloser)
r1 = ret.Get(1).(domain.Variant)
}
if returnFunc, ok := ret.Get(2).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) io.ReadCloser); ok {
r2 = returnFunc(ctx, callerID, id, name)
} else {
if ret.Get(2) != nil {
r2 = ret.Get(2).(io.ReadCloser)
}
}
if returnFunc, ok := ret.Get(2).(func(context.Context, uuid.UUID, uuid.UUID) error); ok {
r2 = returnFunc(ctx, callerID, id)
if returnFunc, ok := ret.Get(3).(func(context.Context, uuid.UUID, uuid.UUID, domain.VariantName) error); ok {
r3 = returnFunc(ctx, callerID, id, name)
} else {
r2 = ret.Error(2)
r3 = ret.Error(3)
}
return r0, r1, r2
return r0, r1, r2, r3
}
// MockAttachmentService_Open_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Open'
type MockAttachmentService_Open_Call struct {
// MockAttachmentService_OpenVariant_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OpenVariant'
type MockAttachmentService_OpenVariant_Call struct {
*mock.Call
}
// Open is a helper method to define mock.On call
// OpenVariant is a helper method to define mock.On call
// - ctx context.Context
// - callerID uuid.UUID
// - id uuid.UUID
func (_e *MockAttachmentService_Expecter) Open(ctx any, callerID any, id any) *MockAttachmentService_Open_Call {
return &MockAttachmentService_Open_Call{Call: _e.mock.On("Open", ctx, callerID, id)}
// - name domain.VariantName
func (_e *MockAttachmentService_Expecter) OpenVariant(ctx any, callerID any, id any, name any) *MockAttachmentService_OpenVariant_Call {
return &MockAttachmentService_OpenVariant_Call{Call: _e.mock.On("OpenVariant", ctx, callerID, id, name)}
}
func (_c *MockAttachmentService_Open_Call) Run(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID)) *MockAttachmentService_Open_Call {
func (_c *MockAttachmentService_OpenVariant_Call) Run(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID, name domain.VariantName)) *MockAttachmentService_OpenVariant_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
@@ -261,21 +268,26 @@ func (_c *MockAttachmentService_Open_Call) Run(run func(ctx context.Context, cal
if args[2] != nil {
arg2 = args[2].(uuid.UUID)
}
var arg3 domain.VariantName
if args[3] != nil {
arg3 = args[3].(domain.VariantName)
}
run(
arg0,
arg1,
arg2,
arg3,
)
})
return _c
}
func (_c *MockAttachmentService_Open_Call) Return(attachment domain.Attachment, readCloser io.ReadCloser, err error) *MockAttachmentService_Open_Call {
_c.Call.Return(attachment, readCloser, err)
func (_c *MockAttachmentService_OpenVariant_Call) Return(attachment domain.Attachment, variant domain.Variant, readCloser io.ReadCloser, err error) *MockAttachmentService_OpenVariant_Call {
_c.Call.Return(attachment, variant, readCloser, err)
return _c
}
func (_c *MockAttachmentService_Open_Call) RunAndReturn(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID) (domain.Attachment, io.ReadCloser, error)) *MockAttachmentService_Open_Call {
func (_c *MockAttachmentService_OpenVariant_Call) RunAndReturn(run func(ctx context.Context, callerID uuid.UUID, id uuid.UUID, name domain.VariantName) (domain.Attachment, domain.Variant, io.ReadCloser, error)) *MockAttachmentService_OpenVariant_Call {
_c.Call.Return(run)
return _c
}
+568
View File
@@ -0,0 +1,568 @@
// Package worker consumes attachment jobs, enforces the lease, produces the
// contract variants and acknowledges only after the durable result is
// committed.
package worker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"os"
"strings"
"sync"
"time"
"github.com/google/uuid"
amqp "github.com/rabbitmq/amqp091-go"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/failpoint"
"mtusi-messenger/internal/imagecheck"
"mtusi-messenger/internal/imageproc"
"mtusi-messenger/internal/platform/objectstore"
)
type Repository interface {
ClaimJob(
ctx context.Context,
message domain.JobMessage,
leaseOwner string,
leaseTTL time.Duration,
) (domain.ClaimResult, error)
Heartbeat(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, leaseTTL time.Duration) (bool, error)
CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, variants []domain.Variant) (bool, error)
RetryJob(
ctx context.Context,
jobID uuid.UUID,
attempt int,
leaseOwner, errorCode string,
nextAvailable time.Time,
sameAttempt bool,
kind string,
) (failed bool, err error)
FailJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner, errorCode string) (bool, error)
MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error
}
type Store interface {
Open(ctx context.Context, key string) (io.ReadCloser, objectstore.Object, error)
Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string, metadata map[string]string) error
Delete(ctx context.Context, key string) error
}
type Consumer interface {
Consume(ctx context.Context, handler broker.Handler) error
Connected() bool
}
type Config struct {
Concurrency int
LeaseTTL time.Duration
Heartbeat time.Duration
JobTimeout time.Duration
RetryBase time.Duration
RetryCap time.Duration
MaxUploadBytes int64
MaxImagePixels int64
Owner string
}
type Worker struct {
repo Repository
store Store
config Config
log *slog.Logger
consumer Consumer
mu sync.RWMutex
running bool
}
func New(repo Repository, store Store, config Config, log *slog.Logger) *Worker {
if log == nil {
log = slog.Default()
}
if config.Owner == "" {
config.Owner = "worker-" + uuid.NewString()
}
return &Worker{repo: repo, store: store, config: config, log: log}
}
// Run starts Concurrency consumer channels. It returns when the context is
// cancelled or one consumer fails; the caller recreates the broker connection.
func (w *Worker) Run(ctx context.Context, consumer Consumer) error {
w.consumer = consumer
w.setRunning(true)
defer w.setRunning(false)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
var (
wg sync.WaitGroup
once sync.Once
firstErr error
)
for range w.config.Concurrency {
wg.Add(1)
go func() {
defer wg.Done()
if err := consumer.Consume(runCtx, w.handle); err != nil && runCtx.Err() == nil {
once.Do(func() {
firstErr = err
cancel()
})
}
}()
}
wg.Wait()
return firstErr
}
func (w *Worker) setRunning(value bool) {
w.mu.Lock()
defer w.mu.Unlock()
w.running = value
}
func (w *Worker) Ready() (consumerRunning, brokerConnected bool) {
w.mu.RLock()
running := w.running
w.mu.RUnlock()
if w.consumer == nil {
return running, false
}
return running, w.consumer.Connected()
}
func (w *Worker) handle(ctx context.Context, delivery amqp.Delivery) broker.Outcome {
message, err := domain.ParseJobMessage(delivery.Body)
if err != nil {
w.log.WarnContext(ctx, "invalid job payload moved to DLQ", slog.Any("error", err))
return broker.Drop
}
result, err := w.repo.ClaimJob(ctx, message, w.config.Owner, w.config.LeaseTTL)
if err != nil {
w.log.WarnContext(ctx, "claim job failed", slog.String("job_id", message.JobID.String()), slog.Any("error", err))
return broker.Requeue
}
switch result.Outcome {
case domain.ClaimAcquired:
w.log.InfoContext(ctx, "job claimed",
slog.String("job_id", message.JobID.String()),
slog.String("attachment_id", message.AttachmentID.String()),
slog.Int("attempt", message.Attempt),
slog.Int("pipeline_version", message.PipelineVersion),
)
case domain.ClaimAlreadyReady, domain.ClaimAlreadyFailed, domain.ClaimSuperseded,
domain.ClaimStaleAttempt, domain.ClaimFutureAttempt, domain.ClaimBusyLease,
domain.ClaimNotDue, domain.ClaimVersionStale, domain.ClaimJobNotFound:
w.log.DebugContext(ctx, "job delivery is a no-op",
slog.String("job_id", message.JobID.String()),
slog.String("outcome", string(result.Outcome)),
)
return broker.Ack
default:
return broker.Ack
}
return w.process(ctx, result)
}
func (w *Worker) process(
ctx context.Context,
result domain.ClaimResult,
) broker.Outcome {
job := result.Job
attachment := result.Attachment
failpoint.Check("WORKER_AFTER_CLAIM", job.ID)
switch failpoint.Mode("WORKER_FAIL", job.ID) {
case "permanent":
return w.fail(ctx, job, permanentError{code: "injected_failure", err: errors.New("injected permanent worker failure")})
case "transient":
return w.fail(ctx, job, errors.New("injected transient worker failure"))
}
jobCtx, cancel := context.WithTimeout(ctx, w.config.JobTimeout)
defer cancel()
leaseLost := false
heartbeatDone := make(chan struct{})
go func() {
defer close(heartbeatDone)
ticker := time.NewTicker(w.config.Heartbeat)
defer ticker.Stop()
for {
select {
case <-jobCtx.Done():
return
case <-ticker.C:
ok, err := w.repo.Heartbeat(jobCtx, job.ID, job.Attempt, w.config.Owner, w.config.LeaseTTL)
if err != nil {
// A DB blip is not proof that the lease is gone. Keep trying;
// the fencing CAS still prevents a stale commit.
w.log.WarnContext(jobCtx, "heartbeat failed", slog.Any("error", err))
continue
}
if !ok {
leaseLost = true
cancel()
return
}
}
}
}()
// The hook runs after the heartbeat starts: it can be paused to test a
// genuinely stuck (not merely slow) worker without consuming every attempt.
failpoint.Sleep("WORKER_SLEEP_AFTER_CLAIM", job.ID)
watchdog := time.AfterFunc(w.config.JobTimeout+30*time.Second, func() {
w.log.ErrorContext(ctx, "worker job watchdog fired",
slog.String("job_id", job.ID.String()),
slog.String("attachment_id", job.AttachmentID.String()),
)
os.Exit(70)
})
defer watchdog.Stop()
err := w.runJob(jobCtx, job, attachment)
cancel()
<-heartbeatDone
if leaseLost {
w.log.WarnContext(ctx, "lease lost; reaper will requeue",
slog.String("job_id", job.ID.String()),
)
return broker.Ack
}
if err == nil {
failpoint.Check("WORKER_AFTER_COMMIT", job.ID)
return broker.Ack
}
return w.fail(ctx, job, err)
}
func (w *Worker) runJob(
ctx context.Context,
job domain.Job,
attachment domain.Attachment,
) error {
params, ok := imageproc.Pipeline(job.PipelineVersion)
if !ok {
return permanentError{code: "unsupported_pipeline", err: fmt.Errorf("pipeline %d", job.PipelineVersion)}
}
sourceKey := job.SourceKey
if sourceKey == "" {
sourceKey = attachment.StagingKey
}
if sourceKey == "" {
sourceKey = attachment.ObjectKey
}
file, size, sum, err := w.spoolSource(ctx, sourceKey)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
defer func() { _ = os.Remove(file.Name()) }()
if size != attachment.SizeBytes || sum != attachment.SHA256 {
return permanentError{code: "source_corrupt", err: errors.New("source bytes do not match metadata")}
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("rewind source: %w", err)
}
if _, err := imagecheck.Validate(file, w.config.MaxImagePixels); err != nil {
return permanentError{code: "invalid_image", err: err}
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("rewind source: %w", err)
}
img, _, err := imageproc.Load(file.Name(), w.config.MaxImagePixels)
if err != nil {
return permanentError{code: "invalid_image", err: err}
}
if sourceKey != attachment.ObjectKey {
if _, err := file.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("rewind original: %w", err)
}
if err := w.store.Put(
ctx, attachment.ObjectKey, file, size, attachment.ContentType,
map[string]string{"sha256": sum},
); err != nil {
return fmt.Errorf("publish original: %w", err)
}
}
variants := make([]domain.Variant, 0, 3)
generated := []func() (imageproc.Image, error){
func() (imageproc.Image, error) { return imageproc.Thumbnail(img, params) },
func() (imageproc.Image, error) { return imageproc.Crop(img, params) },
func() (imageproc.Image, error) { return imageproc.Optimized(img, params) },
}
for idx, generate := range generated {
variantImage, err := generate()
if err != nil {
return permanentError{code: "processing_failed", err: err}
}
key := domain.VariantObjectKey(attachment.ChatID, attachment.ID, job.PipelineVersion, variantImage.Name)
if err := w.store.Put(
ctx, key, bytes.NewReader(variantImage.Data), variantImage.SizeBytes,
variantImage.ContentType, map[string]string{"sha256": variantImage.SHA256},
); err != nil {
return fmt.Errorf("publish variant %s: %w", variantImage.Name, err)
}
if idx == 0 {
failpoint.Check("WORKER_AFTER_FIRST_VARIANT", job.ID)
}
variants = append(variants, domain.Variant{
Name: variantImage.Name,
ContentType: variantImage.ContentType,
SizeBytes: variantImage.SizeBytes,
Width: variantImage.Width,
Height: variantImage.Height,
SHA256: variantImage.SHA256,
ObjectKey: key,
})
}
failpoint.Check("WORKER_AFTER_PUT", job.ID)
completed, err := w.repo.CompleteJob(ctx, job.ID, job.Attempt, w.config.Owner, variants)
if err != nil {
return err
}
if !completed {
w.log.WarnContext(ctx, "completion CAS lost", slog.String("job_id", job.ID.String()))
return nil
}
w.log.InfoContext(ctx, "attachment ready",
slog.String("job_id", job.ID.String()),
slog.String("attachment_id", attachment.ID.String()),
slog.Int("pipeline_version", job.PipelineVersion),
)
if strings.HasPrefix(sourceKey, "staging/") && sourceKey != attachment.ObjectKey {
if err := w.store.Delete(ctx, sourceKey); err != nil && !errors.Is(err, domain.ErrNotFound) {
w.log.WarnContext(ctx, "staging delete failed", slog.String("object_key", sourceKey), slog.Any("error", err))
return nil
}
if sourceKey == attachment.StagingKey {
if err := w.repo.MarkStagingDeleted(ctx, attachment.ID, sourceKey); err != nil {
w.log.WarnContext(ctx, "clear staging projection failed", slog.Any("error", err))
}
}
}
return nil
}
func (w *Worker) spoolSource(ctx context.Context, key string) (*os.File, int64, string, error) {
body, _, err := w.store.Open(ctx, key)
if err != nil {
if errors.Is(err, domain.ErrNotFound) {
return nil, 0, "", permanentError{code: "source_missing", err: err}
}
return nil, 0, "", fmt.Errorf("open source: %w", err)
}
defer body.Close()
file, err := os.CreateTemp("", "lab5-worker-*")
if err != nil {
return nil, 0, "", fmt.Errorf("create worker spool: %w", err)
}
cleanup := func() {
_ = file.Close()
_ = os.Remove(file.Name())
}
hasher := sha256.New()
limited := io.LimitReader(body, w.config.MaxUploadBytes+1)
buffer := make([]byte, 64<<10)
size, err := io.CopyBuffer(io.MultiWriter(file, hasher), limited, buffer)
if err != nil {
cleanup()
return nil, 0, "", fmt.Errorf("copy source: %w", err)
}
if size > w.config.MaxUploadBytes {
cleanup()
return nil, 0, "", permanentError{code: "source_too_large", err: fmt.Errorf("source exceeds %d bytes", w.config.MaxUploadBytes)}
}
return file, size, hex.EncodeToString(hasher.Sum(nil)), nil
}
func (w *Worker) fail(ctx context.Context, job domain.Job, err error) broker.Outcome {
if ctx.Err() != nil {
// Parent cancellation means graceful shutdown. Keep the attempt number:
// an operator restart must not consume the processing budget. The job
// context is already cancelled, so use a short detached context for the
// durable requeue.
retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
failed, retryErr := w.repo.RetryJob(
retryCtx, job.ID, job.Attempt, w.config.Owner, "shutdown",
time.Now().Add(time.Second), true, "shutdown",
)
if retryErr != nil {
return broker.Requeue
}
_ = failed
return broker.Ack
}
code := "processing_failed"
permanent := false
var typed permanentError
if errors.As(err, &typed) {
code = typed.code
permanent = true
}
if errors.Is(err, domain.ErrValidation) || errors.Is(err, domain.ErrUnsupportedMedia) ||
errors.Is(err, domain.ErrTooLarge) || errors.Is(err, domain.ErrConflict) {
permanent = true
if code == "processing_failed" {
code = "invalid_image"
}
}
if errors.Is(err, context.DeadlineExceeded) {
code = "processing_timeout"
permanent = false
}
if permanent {
w.log.WarnContext(ctx, "job permanently failed",
slog.String("job_id", job.ID.String()),
slog.String("error_code", code),
slog.Any("error", err),
)
ok, failErr := w.repo.FailJob(ctx, job.ID, job.Attempt, w.config.Owner, code)
if failErr != nil {
return broker.Requeue
}
if !ok {
return broker.Ack
}
return broker.Drop
}
next := time.Now().Add(w.backoff(job.Attempt))
w.log.WarnContext(ctx, "job retry scheduled",
slog.String("job_id", job.ID.String()),
slog.String("error_code", code),
slog.Time("available_at", next),
slog.Any("error", err),
)
failed, retryErr := w.repo.RetryJob(
ctx, job.ID, job.Attempt, w.config.Owner, code, next, false, "retry",
)
if retryErr != nil {
return broker.Requeue
}
if failed {
return broker.Drop
}
return broker.Ack
}
func (w *Worker) backoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
capDelay := w.config.RetryCap
base := w.config.RetryBase
for range attempt - 1 {
base *= 2
if base >= capDelay {
base = capDelay
break
}
}
if base <= 0 {
return time.Second
}
return time.Duration(rand.Int64N(int64(base))) + time.Millisecond
}
type permanentError struct {
code string
err error
}
func (e permanentError) Error() string { return e.code + ": " + e.err.Error() }
func (e permanentError) Unwrap() error { return e.err }
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Track 2 evidence: versioned pipeline, duplicate delivery, stale-version guard.
Runs N repetitions against the live stand and writes raw CSV/Markdown evidence.
This is a correctness harness, not a load test.
python3 scripts/lab5_track2.py --reps 5
"""
import argparse
import csv
import hashlib
import json
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
from client import Client, Failure, check # noqa: E402
from tests_helpers import compose, jobctl, psql # noqa: E402
FIXTURE = ROOT / 'tests/fixtures/sample.png'
EVIDENCE = ROOT / 'evidence/lab5/track2'
def wait_ready(client, actor, attachment, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
if current['status'] == 'ready':
return current
time.sleep(0.2)
raise Failure('attachment did not become ready')
def wait_version(attachment_id, version, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
row = psql("select pipeline_version || '|' || status from attachments where id='%s'" % attachment_id)
if row == str(version) + '|ready':
return
time.sleep(0.2)
raise Failure('attachment did not reach pipeline version %d' % version)
def variants_by_name(attachment):
return {v['name']: v for v in attachment['variants']}
def scenario_duplicate_before_effect(client, actor, chat):
compose('stop', 'worker')
try:
attached, _ = client.attachment(actor, chat)
time.sleep(1)
job = psql("select job_id from attachment_jobs where attachment_id='%s'" % attached['id'])
check(bool(job), 'job id missing')
jobctl('duplicate', '-job', job)
jobctl('duplicate', '-job', job)
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
ready = wait_ready(client, actor, attached)
names = sorted(variants_by_name(ready))
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate delivery changed the variant set')
check(psql("select count(*) from attachment_variants where attachment_id='%s'" % attached['id']) == '3',
'duplicate delivery created extra variant rows')
return 'duplicate_before_effect', True, 'attempts=' + psql(
"select attempt from attachment_jobs where attachment_id='%s'" % attached['id'])
def scenario_stale_version(client, actor, chat):
attached, content = client.attachment(actor, chat)
v1 = wait_ready(client, actor, attached)
v1_variants = variants_by_name(v1)
original_sha = hashlib.sha256(content).hexdigest()
check(v1['sha256'] == original_sha, 'original checksum changed before reprocess')
v1_job = psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=1" % attached['id'])
check(bool(v1_job), 'v1 job missing')
reprocess = jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2')
check(reprocess.returncode == 0, 'reprocess failed: ' + reprocess.stderr)
wait_version(attached['id'], 2)
v2 = wait_ready(client, actor, attached)
v2_variants = variants_by_name(v2)
check(sorted(v2_variants) == ['crop', 'optimized', 'thumbnail'], 'v2 variant set incomplete')
check(v2_variants['thumbnail']['sha256'] != v1_variants['thumbnail']['sha256'],
'v2 bytes must differ from v1 (pipeline parameters changed)')
check(v2['sha256'] == original_sha, 'original changed during reprocess')
before = json.dumps(v2_variants, sort_keys=True)
# Deliver the old v1 message after v2 is published. It must be acknowledged
# without touching the serving version or its artifacts.
jobctl('duplicate', '-job', v1_job)
time.sleep(2)
after_attachment = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=actor, schema='Attachment')['json']
after_variants = variants_by_name(after_attachment)
check(json.dumps(after_variants, sort_keys=True) == before, 'stale v1 delivery overwrote v2 variants')
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
'serving pipeline version regressed')
check(psql("select status from attachment_jobs where job_id='%s'" % v1_job) == 'ready',
'v1 job state changed unexpectedly')
return 'stale_version_after_v2', True, 'v1_job=' + v1_job
def main():
parser = argparse.ArgumentParser(description='Lab5 track 2 correctness harness')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--reps', type=int, default=3)
args = parser.parse_args()
EVIDENCE.mkdir(parents=True, exist_ok=True)
rows = []
client = Client(args.base_url, None)
for rep in range(1, args.reps + 1):
alice, _, _, chat = client.bootstrap()
for scenario in (scenario_duplicate_before_effect, scenario_stale_version):
name, passed, detail = scenario(client, alice, chat)
rows.append({'scenario': name, 'rep': rep, 'pass': int(passed), 'detail': detail})
print('D%s rep=%d: %s' % (name, rep, 'PASS' if passed else 'FAIL'))
with (EVIDENCE / 'runs.csv').open('w', newline='') as handle:
writer = csv.DictWriter(handle, fieldnames=['scenario', 'rep', 'pass', 'detail'])
writer.writeheader()
writer.writerows(rows)
summary = [
'# Track 2 — versioned pipeline and duplicate/stale delivery',
'',
'Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`,',
'deterministic object keys, a serving version that changes only when all v2',
'variants are committed, and stale-version acknowledgements, an at-least-once',
'pipeline loses no accepted work and never overwrites a newer result with an',
'older duplicate.',
'',
'Method: N=%d repetitions. Each repetition creates fresh users/chat and runs:' % args.reps,
'- duplicate pipeline publication while the worker is stopped, followed by recovery;',
'- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready.',
'',
'Invariants checked: unique variant rows per effect key; ready remains ready;',
'serving pipeline version never regresses; original bytes/checksum unchanged;',
'stale job state is not rewritten.',
'',
'Limitations: this harness covers duplicate and stale/reorder delivery. Crash',
'windows (before/after side effect, before/after ack, outbox gap) and lease',
'takeover are covered by student-tests/lab5_faults.py; both are single-node',
'Compose evidence, not a production availability claim. Delivery is',
'at-least-once; exactly-once is not claimed.',
]
(EVIDENCE / 'summary.md').write_text('\n'.join(summary) + '\n')
passed = sum(row['pass'] for row in rows)
print('PASS: track2 %d/%d scenarios' % (passed, len(rows)))
if __name__ == '__main__':
main()
+13 -7
View File
@@ -1,5 +1,5 @@
#!/bin/sh
# Проверка критерия «LB исключает неготовую реплику» (lab3).
# Проверка критерия «LB исключает неготовую реплику» (lab5).
#
# Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её
# доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и
@@ -23,10 +23,10 @@ if [ -f .env ]; then
set +a
fi
OUT=${OUT:-evidence/lb-readiness.txt}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
OUT=${OUT:-evidence/lab5/lb-readiness-detail.txt}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5}
NET=${PROJECT}_default
IMAGE=${IMAGE:-mtusi-messenger:lab4}
IMAGE=${IMAGE:-mtusi-messenger:lab5}
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
BASE_URL=${BASE_URL:-http://localhost:8080}
CANDIDATE=lb-readiness-candidate
@@ -77,7 +77,7 @@ wait_readiness() {
}
up_slots() {
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }'
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 ~ /^UP/ && $2 != "BACKEND" { printf "%s ", $2 }'
}
wait_new_up_slot() {
@@ -120,7 +120,7 @@ wait_slot_status() {
}
{
echo "# LB excludes a not-ready but reachable replica (lab3)"
echo "# LB excludes a not-ready but reachable replica (lab5)"
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)"
echo
@@ -139,6 +139,11 @@ wait_slot_status() {
-e POSTGRES_USER="${POSTGRES_USER:-messenger}" \
-e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \
-e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \
-e S3_ENDPOINT=http://seaweedfs:8333 -e S3_REGION="${S3_REGION:-us-east-1}" \
-e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" \
-e S3_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID:-lab4app}" \
-e S3_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY:-lab4app-secret}" \
-e S3_USE_PATH_STYLE=true \
"$IMAGE" >/dev/null
if ! wait_readiness 200; then
@@ -170,7 +175,8 @@ wait_slot_status() {
if wait_slot_status "$slot" DOWN; then
echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)"
else
echo " WARNING: $slot is still not DOWN" >&2
echo " FAIL: $slot is still not DOWN" >&2
exit 1
fi
echo " stats after failure: $(stats)"
+10 -10
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -eu
# Restart hook: bounce API replicas and every state store of the lab4 stand.
# Restart hook: bounce API replicas, worker/relay and every state store of the lab5 stand.
#
# scripts/restart.sh
#
@@ -29,16 +29,16 @@ compose() {
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
}
# 1. State stores: restart the running ones, then make sure they are healthy.
# Volumes are attached as before, nothing is dropped or re-seeded.
compose up -d --wait postgres redis seaweedfs
compose restart postgres redis seaweedfs
compose up -d --wait postgres redis seaweedfs
# 1. State stores and broker: restart the running ones, then make sure they are
# healthy. Volumes are attached as before, nothing is dropped or re-seeded.
compose up -d --wait postgres redis seaweedfs broker
compose restart postgres redis seaweedfs broker
compose up -d --wait postgres redis seaweedfs broker
# 2. API replicas: restart in place (same containers, same addresses) and bring
# back any replica that was stopped by the fault-injection hook.
compose restart api
compose up -d --wait --no-deps api
# 2. API replicas, worker and relay: restart in place and bring back any replica
# stopped by a fault-injection hook. The worker reconnects to the broker.
compose restart api worker relay
compose up -d --wait --no-deps api worker relay
# 3. Load balancer keeps its upstream; re-resolve in case addresses changed.
compose restart lb
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps broker
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps worker
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml stop broker
+2 -2
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -eu
# Fault-injection hook: stop exactly ONE api replica of the lab4 stand.
# Fault-injection hook: stop exactly ONE api replica of the lab5 stand.
#
# TARGET_INSTANCE_ID=<hostname|container name|short id> scripts/stop-one.sh
#
@@ -13,7 +13,7 @@ set -eu
# No credentials are printed.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5}
candidates=$(docker ps -q \
--filter "label=com.docker.compose.project=${PROJECT}" \
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
# Fault-injection hook: stop the lab5 worker and leave it stopped.
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml stop worker
+2 -1
View File
@@ -30,7 +30,7 @@ def main():
response = client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote('failover.png'),
201,
202,
actor=alice,
raw=content,
headers={'Content-Type': 'image/png'},
@@ -40,6 +40,7 @@ def main():
instance = response['headers'].get('X-Instance-Id')
check(bool(instance), 'upload response has no X-Instance-Id')
check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'upload checksum mismatch')
attachment = client.wait_ready(alice, attachment)
stop = subprocess.run(
[str(ROOT / 'scripts/stop-one.sh')],
+114 -4
View File
@@ -5,10 +5,12 @@ Run against a live stand:
python3 student-tests/attachments_negative.py --base-url http://localhost:8080
"""
import hashlib
import struct
import sys
import urllib.error
import urllib.request
import uuid
import zlib
from pathlib import Path
from urllib.parse import quote
@@ -19,11 +21,36 @@ from client import Client, Failure, arguments, check, run # noqa: E402
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
def upload(client, actor, chat, content, filename='sample.png', content_type='image/png', expected=201):
def upload(client, actor, chat, content, filename='sample.png', content_type='image/png', expected=202):
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(filename, safe='')
return client.call('POST', path, expected, actor=actor, raw=content, headers={'Content-Type': content_type})
def png_chunk(ctype, payload):
crc = zlib.crc32(ctype + payload) & 0xffffffff
return struct.pack('>I', len(payload)) + ctype + payload + struct.pack('>I', crc)
def crafted_png(width, height, interlace, raster, bit_depth=8, color_type=6):
"""PNG with correct chunk lengths and CRCs around arbitrary IDAT bytes."""
return crafted_png_extra(
width, height, raster, bit_depth=bit_depth, color_type=color_type,
interlace=interlace,
)
def crafted_png_extra(width, height, raster, pre_idat=(), bit_depth=8, color_type=6, interlace=0):
"""PNG whose IDAT is preceded by caller-supplied chunks (PLTE/tRNS/...)."""
ihdr = struct.pack('>IIBBBBB', width, height, bit_depth, color_type, 0, 0, interlace)
return (
b'\x89PNG\r\n\x1a\n'
+ png_chunk(b'IHDR', ihdr)
+ b''.join(pre_idat)
+ png_chunk(b'IDAT', zlib.compress(raster))
+ png_chunk(b'IEND', b'')
)
def main():
args = arguments('Attachment negative scenarios (lab4)')
client = Client(args.base_url, args.ca_file)
@@ -49,6 +76,87 @@ def main():
upload(client, alice, chat, fixture, filename='photo.jpg', content_type='image/jpeg', expected=415)
print('MIME spoof (PNG declared as JPEG): 415')
# Structurally valid PNG (chunk layout, lengths, CRCs all correct) whose
# IDAT payload is not a zlib stream at all. Structure-checking alone must
# not accept it: the stream has to inflate. Regression for a review
# finding that used to produce 202 queued and then failed/invalid_image
# in the worker.
upload(
client, alice, chat,
crafted_png(64, 32, 0, b'this is not a zlib stream'),
expected=415,
)
print('structurally valid PNG with non-zlib IDAT: 415')
# Correct zlib stream that is not a valid PNG raster: an empty payload
# and a scanline with an undefined filter byte must both be 415, not 202.
upload(client, alice, chat, crafted_png(2, 2, 0, b''), expected=415)
print('valid zlib, empty PNG raster: 415')
invalid_filter = b'\x05' + b'\x00' * 8 + b'\x00' + b'\x00' * 8
upload(client, alice, chat, crafted_png(2, 2, 0, invalid_filter), expected=415)
print('valid zlib, invalid PNG scanline filter: 415')
# RGBA forbids bit depth 1; every chunk CRC is correct.
upload(client, alice, chat, crafted_png(1, 1, 0, b'\x00', bit_depth=1), expected=415)
print('invalid PNG color type/bit depth pair: 415')
# Duplicate IHDR is forbidden even with correct CRCs.
ihdr = png_chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 6, 0, 0, 0))
duplicate_header = (
b'\x89PNG\r\n\x1a\n'
+ ihdr + ihdr
+ png_chunk(b'IDAT', zlib.compress(b'\x00' + b'\x00' * 4))
+ png_chunk(b'IEND', b'')
)
upload(client, alice, chat, duplicate_header, expected=415)
print('duplicate PNG IHDR: 415')
# Bytes after the zlib stream inside IDAT are not part of a valid raster.
trailing = (
b'\x89PNG\r\n\x1a\n'
+ png_chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 6, 0, 0, 0))
+ png_chunk(b'IDAT', zlib.compress(b'\x00' + b'\x00' * 4) + b'junk')
+ png_chunk(b'IEND', b'')
)
upload(client, alice, chat, trailing, expected=415)
print('trailing bytes after PNG zlib stream: 415')
# tRNS is forbidden for alpha color types and has fixed lengths otherwise.
rgba_trns = crafted_png_extra(
1, 1, b'\x00' + b'\x00' * 4,
pre_idat=(png_chunk(b'tRNS', b'\x00'),), color_type=6,
)
upload(client, alice, chat, rgba_trns, expected=415)
print('tRNS on RGBA PNG: 415')
rgb_trns = crafted_png_extra(
1, 1, b'\x00' + b'\x00' * 3,
pre_idat=(png_chunk(b'tRNS', b'\x00'),), color_type=2,
)
upload(client, alice, chat, rgb_trns, expected=415)
print('short tRNS on RGB PNG: 415')
# A 1-bit palette cannot address three entries.
palette_overflow = crafted_png_extra(
1, 1, b'\x00\x00',
pre_idat=(png_chunk(b'PLTE', bytes((255, 0, 0, 0, 255, 0, 0, 0, 255))),),
bit_depth=1, color_type=3,
)
upload(client, alice, chat, palette_overflow, expected=415)
print('PLTE entries exceed palette bit depth: 415')
# Palette transparency cannot describe more entries than PLTE provides.
palette_trns = crafted_png_extra(
1, 1, b'\x00\x00',
pre_idat=(
png_chunk(b'PLTE', bytes((255, 0, 0))),
png_chunk(b'tRNS', b'\x00\x00'),
),
bit_depth=8, color_type=3,
)
upload(client, alice, chat, palette_trns, expected=415)
print('palette tRNS exceeds PLTE entries: 415')
# Dangerous display names.
for name in ('../x.png', 'a/b.png', 'a\\b.png', 'a\x00.png', 'a\n.png'):
upload(client, alice, chat, fixture, filename=name, expected=400)
@@ -92,16 +200,18 @@ def main():
)
print('duplicate attachment ids: 400')
# The only variant in the lab4 contract is original; other values are 400.
# Lab5 exposes the derived variants after ready; queued content is a conflict.
own, content = client.attachment(alice, chat)
client.call('GET', '/api/v1/attachments/' + own['id'] + '/content?variant=thumbnail', 400, actor=alice)
client.call('GET', '/api/v1/attachments/' + own['id'] + '/content', 409, actor=alice)
own = client.wait_ready(alice, own)
client.call('GET', '/api/v1/attachments/' + own['id'] + '/content?variant=thumbnail', 200, actor=alice)
# Attachment-only message is valid.
client.message(alice, chat, '', attachments=[own['id']])
print('attachment-only message: 201')
# Object keys are server-generated; anonymous direct access must be denied.
key = 'attachments/' + chat['id'] + '/' + own['id']
key = 'attachments/' + chat['id'] + '/' + own['id'] + '/original'
request = urllib.request.Request('http://127.0.0.1:8333/messenger-attachments/' + key, method='GET')
try:
urllib.request.urlopen(request, timeout=5)
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Lab5: accepted work survives a stopped worker and a broker restart.
Run against a live stand:
python3 student-tests/lab5_async.py --base-url http://localhost:8080
"""
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from client import Client, Failure, arguments, check, run # noqa: E402
from tests_helpers import ROOT, compose, jobctl, psql # noqa: E402
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
def wait_api_ready(client, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
client.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.2)
raise Failure('API did not become ready after restart')
def wait_state(client, actor, attachment, expected, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
if current['status'] == expected:
return current
time.sleep(0.2)
raise Failure('attachment did not reach ' + expected)
def main():
args = arguments('Lab5 async worker/broker durability')
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
content = FIXTURE.read_bytes()
# 1. Worker down: API still accepts 202, content is 409, the message links.
compose('stop', 'worker')
try:
attached, _ = client.attachment(alice, chat)
check(attached['status'] == 'queued', '202 snapshot must be queued')
queued = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
check(queued['status'] == 'queued', 'worker stopped but attachment is not queued')
client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice)
client.message(alice, chat, '', attachments=[attached['id']])
time.sleep(2)
still = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
check(still['status'] == 'queued', 'worker stopped but job was processed synchronously')
# The README demo restarts the API after the 202 and before the worker.
compose('restart', 'api')
wait_api_ready(client)
print('worker down: 202 queued, content 409, queued attachment linked, API restarted')
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
ready = client.wait_ready(alice, attached)
thumb = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=thumbnail', actor=bob)
check(thumb['headers'].get_content_type() == 'image/jpeg', 'thumbnail content type mismatch')
download = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
check(download['bytes'] == content, 'original bytes changed')
check(psql("select coalesce(staging_key, '') from attachments where id='%s'" % ready['id']) == '',
'staging key was not cleared after ready')
print('worker restarted: ready, thumbnail and original downloadable, staging cleared')
# 2. Broker restart with its volume: an intent accepted while the broker is
# down is published by the relay after recovery.
compose('stop', 'relay')
compose('stop', 'broker')
try:
during, _ = client.attachment(alice, chat)
check(during['status'] == 'queued', 'upload during broker outage must be queued')
print('broker down: upload accepted into outbox')
finally:
compose('up', '-d', '--wait', '--no-deps', 'broker', check=False)
compose('up', '-d', '--no-deps', 'relay', check=False)
ready = client.wait_ready(alice, during, wait_seconds=90)
print('broker + relay recovered: queued job reached ready')
# 3. A full stack restart with volumes keeps ready data and the worker path.
compose('restart', 'api', 'worker', 'relay')
wait_api_ready(client)
recovered = client.call('GET', '/api/v1/attachments/' + ready['id'], actor=bob, schema='Attachment')['json']
check(recovered['status'] == 'ready', 'ready attachment lost across restart')
body = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
check(body['bytes'] == content, 'original lost across restart')
print('stack restart: ready attachment and original preserved')
# 4. Reprocess of a queued upload while the worker is down. The final
# original has never been written, so pipeline 2 must source from staging;
# pointing at the not-yet-existing final original superseded v1 and left
# the attachment queued forever with v2 failed (source_missing).
compose('stop', 'worker')
try:
pinned, _ = client.attachment(alice, chat)
check(pinned['status'] == 'queued', 'reprocess fixture must be queued')
result = jobctl('reprocess', '-attachment', pinned['id'], '-pipeline', '2')
check('SourceKey' in result.stdout, 'reprocess did not return the created job')
queued2 = client.call('GET', '/api/v1/attachments/' + pinned['id'], actor=alice, schema='Attachment')['json']
check(queued2['status'] == 'queued', 'reprocessed upload is not queued')
check(psql("select count(*) from attachment_job_events e join attachment_jobs j on j.job_id=e.job_id where j.attachment_id='%s' and j.pipeline_version=1 and e.event='superseded'" % pinned['id']) == '1',
'reprocess did not record the v1 superseded event')
print('worker down: reprocess pipeline 2 accepted, v1 audit event recorded, upload still queued')
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
ready2 = client.wait_ready(alice, pinned, wait_seconds=90)
check(psql("select pipeline_version from attachments where id='%s'" % pinned['id']) == '2',
'serving pipeline version is not 2 after reprocess')
check(psql("select status from attachment_jobs where attachment_id='%s' order by pipeline_version" % pinned['id']) == 'superseded\nready',
'old job not superseded or new job not ready')
print('worker up: reprocessed upload reached ready at pipeline 2, v1 superseded')
# 5. A pipeline version the worker cannot execute must be refused by the
# operator CLI before any state change; the accepted v1 upload keeps its
# job and finishes normally instead of being superseded into queued.
compose('stop', 'worker')
try:
preserved, _ = client.attachment(alice, chat)
check(preserved['status'] == 'queued', 'unsupported-pipeline fixture must be queued')
refused = jobctl('reprocess', '-attachment', preserved['id'], '-pipeline', '3', check=False)
check(refused.returncode != 0 and 'unsupported pipeline version 3' in refused.stderr,
'unsupported pipeline version was not rejected')
check(psql("select status from attachment_jobs where attachment_id='%s' and pipeline_version=1" % preserved['id']) == 'queued',
'v1 job changed by rejected reprocess')
print('worker down: reprocess pipeline 3 refused, v1 job untouched')
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
client.wait_ready(alice, preserved, wait_seconds=90)
check(psql("select pipeline_version from attachments where id='%s'" % preserved['id']) == '1',
'rejected reprocess changed serving version')
print('worker up: preserved upload reached ready at pipeline 1')
print('PASS: lab5 async worker/broker durability')
if __name__ == '__main__':
try:
run(main)
finally:
compose('up', '-d', '--no-deps', 'worker', 'relay', check=False)
+325
View File
@@ -0,0 +1,325 @@
#!/usr/bin/env python3
"""Lab5 level 4: retry, DLQ/replay, crash windows, lease takeover.
The stand must be running. The script temporarily recreates worker/relay with
FAULTS_ENABLED=1 and targeted failpoints, then restores the normal services.
python3 student-tests/lab5_faults.py --base-url http://localhost:8080
"""
import json
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from client import Client, Failure, arguments, check, run # noqa: E402
from tests_helpers import compose, jobctl, psql # noqa: E402
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
WORKER = 'mtusi-messenger-lab5-worker-1'
RELAY = 'mtusi-messenger-lab5-relay-1'
def worker_with(**env):
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=env)
def relay_with(**env):
compose('up', '-d', '--force-recreate', '--no-deps', 'relay', env=env)
def reset_worker():
worker_with()
def reset_relay():
relay_with()
def job_id(attachment_id):
value = psql("select job_id from attachment_jobs where attachment_id = '%s'" % attachment_id)
check(bool(value), 'job row is missing for ' + attachment_id)
return value.splitlines()[0]
def wait_db(attachment_id, predicate, timeout=120, description=''):
deadline = time.monotonic() + timeout
last = ''
while time.monotonic() < deadline:
rows = psql("select a.status || '|' || coalesce(a.error_code,'') || '|' || j.attempt from attachments a join attachment_jobs j on j.attachment_id=a.id where a.id='%s'" % attachment_id)
if rows:
last = rows.splitlines()[0]
status, error, attempt = last.split('|')
if predicate(status, error, int(attempt)):
return status, error, int(attempt)
time.sleep(0.2)
raise Failure('timeout waiting for ' + description + ' (last=' + last + ')')
def queue_messages(name):
project = 'mtusi-messenger-lab5'
result = subprocess.run(
['docker', 'exec', project + '-broker-1', 'rabbitmqctl', '-p', 'messenger', 'list_queues', 'name', 'messages'],
capture_output=True, text=True, check=True,
)
for line in result.stdout.splitlines():
if line.startswith(name):
return int(line.split()[1])
return 0
def scenario_permanent_failure_and_replay(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='permanent')
status, code, attempts = wait_db(attached['id'], lambda s, e, a: s == 'failed', 40, 'permanent failure')
check(code == 'injected_failure', 'unexpected permanent error_code: ' + code)
check(attempts == 1, 'a permanent error must not consume retries')
meta = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
check(meta['status'] == 'failed' and meta['error_code'] == code, 'API did not expose failed/error_code')
client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice)
client.call(
'POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text': '', 'attachment_ids': [attached['id']]},
headers={'Idempotency-Key': 'failed-link-' + attached['id']},
)
failed_view = jobctl('failed', '-limit', '20').stdout
check(jid in failed_view, 'failed job is missing from the operator registry')
print('permanent failure: failed/' + code + ', content 409, failed attachment not linkable, registry visible')
reset_worker()
replay = jobctl('replay', '-job', jid)
check(replay.returncode == 0, 'jobctl replay failed: ' + replay.stderr)
ready = client.wait_ready(alice, attached, wait_seconds=60)
check(len([v for v in ready['variants'] if v['name'] == 'thumbnail']) == 1, 'replay created a duplicate thumbnail')
print('replay: failed -> queued -> ready, one thumbnail')
# Replay of a ready job must be refused and must not create another effect.
again = jobctl('replay', '-job', jid, check=False)
check(again.returncode != 0, 'replay of a ready job must be refused')
print('replay ready job: refused')
def scenario_transient_exhaustion(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
dlq_before = queue_messages('attachment.process.dead')
worker_with(
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='transient',
PROCESSING_RETRY_BASE='100ms', PROCESSING_RETRY_CAP='500ms',
)
status, code, attempts = wait_db(attached['id'], lambda s, e, a: s == 'failed', 120, 'retry exhaustion')
check(attempts == 5, 'retry cap must be exactly 5, got ' + str(attempts))
check(code == 'retry_exhausted', 'unexpected exhaustion code: ' + code)
events = psql("select event || ':' || coalesce(attempt,0) from attachment_job_events where job_id='%s' order by id" % jid)
check(events.count('retried:') == 4, 'expected four managed retries: ' + events)
check(events.count('failed:') == 1, 'expected one terminal failure: ' + events)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
if queue_messages('attachment.process.dead') >= dlq_before + 1:
break
time.sleep(0.2)
check(queue_messages('attachment.process.dead') >= dlq_before + 1,
'terminal failure did not add a DLQ message')
print('transient exhaustion: attempts=5, retry_exhausted, four retries, DLQ message present')
reset_worker()
def scenario_crash_after_first_variant(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
relay_with(OUTBOX_REAPER_INTERVAL='2s')
worker_with(
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_FIRST_VARIANT='exit',
PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='10s',
)
ready = client.wait_ready(alice, attached, wait_seconds=90)
status, _, attempts = wait_db(attached['id'], lambda s, e, a: s == 'ready', 10, 'crash recovery')
check(attempts >= 2, 'crash recovery must use a new managed attempt, got ' + str(attempts))
names = sorted(v['name'] for v in ready['variants'])
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate/partial variant set: ' + str(names))
print('crash after first variant: redelivered, same three variants, attempts=' + str(attempts))
reset_worker()
reset_relay()
def scenario_crash_after_commit(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_COMMIT='exit')
ready = client.wait_ready(alice, attached, wait_seconds=60)
check(ready['status'] == 'ready', 'job did not stay ready after commit/ack crash')
check(psql("select attempt from attachment_jobs where job_id='%s'" % jid) == '1',
'redelivery after ready must be a no-op, not a new attempt')
print('crash after commit before ack: ready remains, redelivery is a no-op')
reset_worker()
def scenario_crash_before_side_effect(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
relay_with(OUTBOX_REAPER_INTERVAL='2s')
worker_with(
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_CLAIM='exit',
PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='12s',
)
ready = client.wait_ready(alice, attached, wait_seconds=90)
status, _, attempts = wait_db(attached['id'], lambda s, e, a: s == 'ready', 10, 'pre-effect crash recovery')
check(attempts >= 2, 'pre-effect crash must use a new managed attempt, got ' + str(attempts))
check(sorted(v['name'] for v in ready['variants']) == ['crop', 'optimized', 'thumbnail'],
'pre-effect crash produced an incomplete variant set')
print('crash after claim before side effect: requeued, attempts=' + str(attempts) + ', ready')
reset_worker()
reset_relay()
def scenario_outbox_gap(client, alice, chat):
compose('stop', 'relay')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
unpublished = psql("select count(*) from outbox_messages where job_id='%s' and published_at is null" % jid)
check(unpublished == '1', 'upload did not leave a durable outbox intent')
relay_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_OUTBOX_AFTER_COMMIT='exit')
ready = client.wait_ready(alice, attached, wait_seconds=90)
check(psql("select count(*) from outbox_messages where job_id='%s' and published_at is null" % jid) == '0',
'outbox row stayed unpublished after recovery')
print('outbox gap: crash before publish recovered through the relay, outbox published, ready')
reset_relay()
def scenario_lease_takeover(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid = job_id(attached['id'])
relay_with(OUTBOX_REAPER_INTERVAL='1s')
worker_with(
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_SLEEP_AFTER_CLAIM='8s',
PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='12s',
)
wait_db(attached['id'], lambda s, e, a: s == 'processing', 20, 'processing claim')
subprocess.run(['docker', 'pause', WORKER], check=True, capture_output=True)
wait_db(attached['id'], lambda s, e, a: s == 'queued' and a >= 2, 30, 'lease expiry requeue')
subprocess.run(['docker', 'unpause', WORKER], check=True, capture_output=True)
ready = client.wait_ready(alice, attached, wait_seconds=60)
check(len(ready['variants']) == 3, 'lease takeover produced an incomplete variant set')
print('lease expiry: paused worker, reaper requeued, stale attempt fenced, ready')
reset_worker()
reset_relay()
def scenario_replay_reprocessed_version(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
reprocessed = json.loads(jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2').stdout)
jid = reprocessed['ID']
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='permanent')
deadline = time.monotonic() + 40
last = ''
while time.monotonic() < deadline:
job_status = psql("select status from attachment_jobs where job_id='%s'" % jid)
last = psql("select status || '|' || coalesce(error_code,'') from attachments where id='%s'" % attached['id'])
if job_status == 'failed' and last.startswith('failed|'):
break
time.sleep(0.2)
check(job_status == 'failed' and last == 'failed|injected_failure',
'reprocessed failure did not settle: ' + last)
compose('stop', 'worker')
jobctl('replay', '-job', jid)
projection = client.call(
'GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment',
)['json']
check(projection['status'] == 'queued' and projection['error_code'] is None,
'replay left stale terminal projection: ' + str(projection['status']))
reset_worker()
client.wait_ready(alice, attached, wait_seconds=90)
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
'replayed reprocess did not publish pipeline 2')
print('replay of failed reprocess: queued projection restored, then ready at pipeline 2')
def scenario_replay_older_version_refused(client, alice, chat):
compose('stop', 'worker')
attached, _ = client.attachment(alice, chat)
jid_v1 = job_id(attached['id'])
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid_v1, FAULT_WORKER_FAIL='permanent')
status, code, _ = wait_db(attached['id'], lambda s, e, a: s == 'failed', 40, 'failed v1')
check(code == 'injected_failure', 'unexpected v1 failure: ' + code)
compose('stop', 'worker')
reprocessed = json.loads(jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2').stdout)
refused = jobctl('replay', '-job', jid_v1, check=False)
check(refused.returncode != 0, 'older failed version was replayed while v2 exists')
jobs = psql("select pipeline_version || ':' || status from attachment_jobs where attachment_id='%s' order by pipeline_version" % attached['id'])
check(jobs == '1:failed\n2:queued', 'older replay changed job state: ' + jobs.replace('\n', ', '))
print('replay older failed version while v2 queued: refused, state intact')
reset_worker()
client.wait_ready(alice, attached, wait_seconds=90)
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
'v2 did not publish after older replay was refused')
check(psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=2" % attached['id']) == reprocessed['ID'],
'v2 job identity changed')
print('v2 published after older replay refusal')
def main():
args = arguments('Lab5 level 4 fault injection')
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
scenarios = [
scenario_permanent_failure_and_replay,
scenario_replay_older_version_refused,
scenario_replay_reprocessed_version,
scenario_transient_exhaustion,
scenario_crash_after_first_variant,
scenario_crash_after_commit,
scenario_crash_before_side_effect,
scenario_outbox_gap,
scenario_lease_takeover,
]
try:
for scenario in scenarios:
scenario(client, alice, chat)
finally:
reset_worker()
reset_relay()
print('PASS: lab5 level 4 fault injection')
if __name__ == '__main__':
run(main)
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Lab5: metadata, bytes and dimensions of all contract variants.
Run against a live stand:
python3 student-tests/lab5_variants.py --base-url http://localhost:8080
"""
import hashlib
import struct
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from client import Client, Failure, arguments, check, run # noqa: E402
from image_probe import jpeg_size # noqa: E402
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
def webp_size(data):
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'not a WebP container')
chunk = data[12:16]
if chunk == b'VP8X':
width = 1 + int.from_bytes(data[24:27], 'little')
height = 1 + int.from_bytes(data[27:30], 'little')
return width, height
if chunk == b'VP8 ':
start = data.index(b'\x9d\x01\x2a', 12) + 3
width = int.from_bytes(data[start:start + 2], 'little') & 0x3fff
height = int.from_bytes(data[start + 2:start + 4], 'little') & 0x3fff
return width, height
if chunk == b'VP8L':
bits = int.from_bytes(data[21:25], 'little')
width = (bits & 0x3fff) + 1
height = ((bits >> 14) & 0x3fff) + 1
return width, height
raise Failure('unsupported WebP chunk ' + repr(chunk))
def main():
args = arguments('Lab5 variant contract')
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attached, content = client.attachment(alice, chat)
ready = client.wait_ready(alice, attached)
by_name = {variant['name']: variant for variant in ready['variants']}
check(set(by_name) == {'thumbnail', 'crop', 'optimized'}, 'unexpected variant set: ' + str(sorted(by_name)))
expected = {
'thumbnail': ('image/jpeg', (256, 160)),
'crop': ('image/jpeg', (128, 128)),
'optimized': ('image/webp', (320, 200)),
}
for name, (content_type, dims) in expected.items():
variant = by_name[name]
response = client.call(
'GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=' + name, actor=bob,
)
data = response['bytes']
check(response['headers'].get_content_type() == content_type, name + ' content type mismatch')
check(variant['content_type'] == content_type, name + ' metadata content type mismatch')
check(len(data) == variant['size_bytes'], name + ' size_bytes mismatch')
check(hashlib.sha256(data).hexdigest() == variant['sha256'], name + ' sha256 mismatch')
if name == 'optimized':
actual = webp_size(data)
check(actual == dims, name + ' dimensions ' + str(actual))
check(b'EXIF' not in data[:64], 'optimized WebP must not carry EXIF')
else:
actual = jpeg_size(data)
check(actual == dims, name + ' dimensions ' + str(actual))
check('Exif' not in data[:64].decode('latin-1', 'ignore'), name + ' must not carry EXIF')
check((variant['width'], variant['height']) == dims, name + ' metadata dimensions mismatch')
original = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
check(original['bytes'] == content, 'original bytes changed')
check(original['headers'].get_content_type() == 'image/png', 'original content type changed')
print('variants: thumbnail 256x160, crop 128x128, optimized WebP 320x200; metadata == bytes')
print('PASS: lab5 variant contract')
if __name__ == '__main__':
run(main)
+5 -3
View File
@@ -52,10 +52,10 @@ def put(url, content, content_type='image/png'):
return error.code
def complete(client, actor, attachment_id, digest, expected=200):
def complete(client, actor, attachment_id, digest, expected=202):
return client.call(
'POST', '/api/v1/attachments/' + attachment_id + '/complete', expected,
actor=actor, body={'sha256': digest}, schema='Attachment' if expected == 200 else None,
actor=actor, body={'sha256': digest}, schema='Attachment' if expected == 202 else None,
)
@@ -73,7 +73,8 @@ def main():
check(put(presigned['upload_url'], content) == 200, 'direct PUT failed')
attachment = complete(client, alice, presigned['attachment_id'], digest)['json']
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(content), 'completed metadata mismatch')
print('happy path: direct PUT 200, complete ready, checksum/size match')
attachment = client.wait_ready(alice, attachment)
print('happy path: direct PUT 200, complete queued, ready after worker')
client.message(alice, chat, '', attachments=[attachment['id']])
download = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob)
@@ -83,6 +84,7 @@ def main():
# Completion is idempotent.
again = complete(client, alice, presigned['attachment_id'], digest)['json']
check(again['id'] == attachment['id'], 'repeated complete returned another attachment')
check(again['status'] in ('queued', 'processing', 'ready'), 'repeated complete returned an invalid state')
print('repeated complete is idempotent')
# A foreign user cannot complete someone else's pending upload.
+20 -3
View File
@@ -10,15 +10,32 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, check=True):
def compose(*args, check=True, env=None):
command = [*os.environ.get('COMPOSE_BIN', 'docker compose').split(), '--env-file', '.env', '-f', 'compose.yaml', *args]
environment = {**os.environ, **(env or {})}
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=check)
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=check, env=environment)
def jobctl(*args, check=True):
"""Run the operator CLI inside the worker image."""
return compose('run', '--rm', '--entrypoint', '/usr/local/bin/jobctl', 'worker', *args, check=check)
def psql(sql):
"""Read a scalar/list from the stand database (operator evidence only)."""
project = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-messenger-lab5')
result = subprocess.run(
['docker', 'exec', project + '-postgres-1', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql],
cwd=ROOT, capture_output=True, text=True, check=True,
)
return result.stdout.strip()
def containers_by_hostname(service='api'):
"""{hostname: {id, name, status}} для контейнеров сервиса."""
project = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-messenger-lab4')
project = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-messenger-lab5')
listed = subprocess.run(
[
'docker', 'ps', '-q',
+3 -3
View File
@@ -1,6 +1,6 @@
// Command specgen produces the OpenAPI document used for code generation.
//
// The mandatory contract (contracts/openapi.json, version 1.4.0) describes the
// The mandatory contract (contracts/openapi.json, version 1.5.0) describes the
// cross-field rule "text is non-empty OR attachment_ids is non-empty" with an
// anyOf. ogen v1.24 cannot generate Go types for two unnamed anyOf branches
// with the same name; ignoring the limitation silently drops entire operations.
@@ -89,8 +89,8 @@ func derive() ([]byte, error) {
}
number, _ := lab["number"].(float64)
if number != 4 {
return nil, fmt.Errorf("%s: expected lab number 4, got %v", labPath, lab["number"])
if number != 5 {
return nil, fmt.Errorf("%s: expected lab number 5, got %v", labPath, lab["number"])
}
for _, path := range rewriteTargets {