diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8e0d5d5..f2d5cad 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh - name: Lint uses: golangci/golangci-lint-action@v7 @@ -80,6 +80,8 @@ jobs: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 docker compose up -d --scale api=2 --wait --wait-timeout 90 + make test-lab5-acceptance + make test-lab5 - name: Container logs if: failure() diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 36d9be7..54da2be 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -60,7 +60,7 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh - name: Lint uses: golangci/golangci-lint-action@v7 @@ -93,6 +93,8 @@ jobs: go run ./cmd/dependencycheck -base-url=http://nginx:8080 make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + make test-lab5-acceptance + make test-lab5 - name: Publish verified images id: publish diff --git a/Dockerfile b/Dockerfile index ff27469..efd6b0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . RUN go build -p=1 -o /out/messenger ./cmd/messenger -RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment +RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment FROM build AS test @@ -35,7 +35,7 @@ FROM alpine:3.24.2 RUN addgroup -S app && adduser -S -G app -u 10001 app COPY --from=build --chown=app:app /out/messenger /app/messenger -COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment +COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment USER app WORKDIR /app diff --git a/Makefile b/Makefile index 5d86d17..43e0a2a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,9 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-lab5: + ./scripts/test-lab5.sh + +test-lab5-acceptance: + ./scripts/test-lab5-acceptance.sh diff --git a/REPORT.md b/REPORT.md index e98f75e..a3422fd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,182 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Пластинин Денис, БВТ2303. +- Целевая оценка: 5. +- Выбранный трек: 2, доказанная идемпотентность versioned pipeline. +- Ветка решения: `solution/bvt2303-plastinin`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Обычные настройки приложения находятся в `config/messenger.yaml`; `.env` содержит адреса окружения, локальные credentials и параметры контейнеров. -## Архитектура и решения +```sh +cp lab5.env.example .env +docker compose config --quiet +docker compose up -d --build --scale api=2 --wait --wait-timeout 180 +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +После запуска доступны: + +- API через Nginx: `http://localhost:8080`; +- Swagger UI: `http://localhost:8081`; +- Prometheus: `http://localhost:9090`; +- Grafana: `http://localhost:3000`, пользователь `admin`, пароль берётся из `GRAFANA_ADMIN_PASSWORD`. + +Интеграционные проверки PostgreSQL и Valkey запускаются отдельно: + +```sh +docker compose --profile tests run --rm --build tests +``` + +Эксперимент трека 5 временно останавливает обычные dispatcher и worker, запускает управляемые сбои, затем возвращает процессы в работу: + +```sh +make test-lab5 +``` + +Обязательный сценарий с остановленным worker и разрывом commit → publish: + +```sh +make test-lab5-acceptance +``` + +Обычная остановка сохраняет volumes: + +```sh +docker compose down +``` + +Намеренное удаление всех учебных данных: + +```sh +docker compose down -v +``` + +## Архитектура + +```mermaid +flowchart LR + Client --> Nginx + Nginx --> API1[API replica 1] + Nginx --> API2[API replica 2] + API1 --> PostgreSQL + API2 --> PostgreSQL + API1 --> Valkey[Valkey sessions] + API2 --> Valkey + API1 --> S3[(MinIO / S3)] + API2 --> S3 + PostgreSQL -->|outbox lease| Dispatcher + Dispatcher -->|XADD| Redis[(Redis Streams + AOF)] + Redis -->|XREADGROUP| Worker + Worker --> S3 + Worker -->|lease + result transaction| PostgreSQL + Worker -->|XACK after commit| Redis +``` + +PostgreSQL является источником истины для metadata, состояния job, попытки, lease, версий pipeline, вариантов и outbox. MinIO хранит исходник в staging и готовые объекты. Valkey хранит только сессии. Отдельный Redis используется как durable broker: включены AOF, `appendfsync always`, volume и политика `noeviction`. + +Temporal не используется. В работе один короткий тип workflow без ожидания внешних событий и компенсаций между множеством сервисов. Transactional outbox, job lease и Redis consumer group закрывают требуемые окна отказа с меньшим числом компонентов. Добавление Temporal потребовало бы Temporal Service и отдельного persistent store, при этом PostgreSQL job/outbox всё равно понадобился бы для атомарной связи с attachment metadata. + +## Граница ответа 202 + +API полностью читает и проверяет PNG/JPEG: MIME, фактический формат, размер и число пикселей. Затем создаётся `pending` metadata, а байты записываются в приватный ключ `staging//` в S3. После успешной записи одна транзакция PostgreSQL переводит attachment в `queued`, создаёт job и outbox event. Только после commit клиент получает `202 Accepted` со snapshot состояния `queued`. + +Если запись staging не удалась, `202` не возвращается. Если соединение потеряно после commit, сохранённый outbox остаётся источником восстановления. Бинарные данные, S3 ключи и credentials в Redis не передаются: envelope содержит только schema version, job ID, attachment ID, pipeline version, attempt, время и trace context. + +## Обработка изображений + +Worker повторно проверяет доверенный staging объект и создаёт четыре content-addressed объекта: + +| Вариант | Формат | Правило | +| --- | --- | --- | +| `original` | исходный PNG/JPEG | байты не изменяются | +| `thumbnail` | JPEG quality 85 | fit в `256×256`, без увеличения | +| `crop` | JPEG quality 85 | центральный квадрат `128×128`, увеличение разрешено | +| `optimized` | WebP quality 82, method 4 | fit в `1280×1280`, без увеличения | + +EXIF orientation применяется к производным вариантам. Прозрачность сводится на белый фон. Повторное кодирование удаляет metadata. Ключ эффекта имеет вид `attachments//pipelines///.`, поэтому повтор той же операции записывает тот же объект. + +Concurrency и prefetch задаются через `worker.concurrency` и `worker.prefetch`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `worker.memory_bytes` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд. Worker продлевает lease каждые 5 секунд; потеря lease отменяет обработку. Decode и собственные циклы преобразования проверяют context во время работы. + +## Доставка, retry и восстановление + +Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль. Опубликованные события хранятся семь суток и удаляются раз в минуту пакетами до 1000 строк, не задерживая первый dispatch после запуска. + +Worker читает consumer group через `XREADGROUP`. При старте обработки PostgreSQL атомарно проверяет `job_id`, `pipeline_version`, `attempt`, выдаёт lease token и переводит attachment в `processing`. Результат применяется только при совпадении всех fencing полей и живом lease. `XACK` выполняется после commit результата. После ACK обработанная запись удаляется из основного stream; pending записи не вытесняются. Failed stream ограничен приблизительно 10 000 записями. + +Временная ошибка переводит job обратно в `queued`, увеличивает `attempt` и в той же транзакции создаёт новый outbox event. Задержка растёт экспоненциально и получает детерминированный jitter до 25%. Постоянные ошибки (`invalid_image`, `image_too_large`, `source_missing`) и исчерпание пяти попыток переводят attachment в `failed`. Явный replay выполняется командой: + +```sh +docker compose run --rm api /app/messenger replay-attachment +``` + +Replay допустим только для `failed`, увеличивает attempt и сохраняет audit record. Повтор `ready` job становится no-op. `XAUTOCLAIM` возвращает зависшие Redis deliveries, а PostgreSQL recovery возвращает job с истёкшим lease в очередь. Старый consumer не может подтвердить delivery после передачи новому consumer: Lua script проверяет владельца PEL перед `XACK` или переносом в failed stream. + +## Эксперимент трека 5 + +### Гипотеза + +At-least-once доставка не создаст второй логический результат, если effect адресуется по `(attachment_id, pipeline_version, variant)`, а публикация metadata защищена `attempt` и lease token. Старая версия pipeline не сможет перезаписать текущую. + +### Методика + +Go-программа `cmd/lab5experiment` работает с настоящими PostgreSQL, Redis Streams и MinIO из Compose. Используется PNG `320×200`. Обычные worker и dispatcher на время прогона остановлены, чтобы порядок событий задавался экспериментом. + +Автоматически выполняются: + +1. crash после claim, но до side effect; +2. crash после записи всех S3 объектов, но до commit результата; +3. crash после commit результата, но до consumer ACK; +4. две дополнительные доставки уже завершённой job; +5. постоянная ошибка повреждённого staging и явный replay; +6. завершение pipeline v2, после которого доставляется устаревшая job v1. + +Для первых двух сценариев ожидается истечение lease, увеличение attempt и завершение следующей попыткой. В третьем и четвёртом сценариях повтор должен подтвердиться как no-op. Повреждённый staging дважды завершается одинаковым безопасным `invalid_image`, причём replay увеличивает attempt и проходит только через audit/outbox. После готовой версии v2 поздняя v1 не меняет metadata. + +### Результат + +Raw summaries сохранены в `evidence/lab5-idempotency.json` и `evidence/lab5-resilience.json`. + +| Воздействие | До восстановления | После восстановления | +| --- | --- | --- | +| crash до side effect | 0 конечных объектов | attempt 2, `ready`, 4 объекта | +| crash после side effect | 4 объекта, DB ещё `processing` | attempt 2, `ready`, те же 4 объекта | +| crash после commit до ACK | DB уже `ready`, 4 объекта | attempt 1, повтор no-op, 4 объекта | +| две duplicate deliveries | 4 объекта | 4 объекта | +| permanent failure и replay | `failed`, `invalid_image` | replay attempt 2, снова `failed`, `invalid_image` | +| stale v1 после готовой v2 | `ready` v2 | остаётся `ready` v2, 8 объектов для двух версий | + +Все пять подтверждённых logical jobs дошли до терминального состояния: четыре `ready`, одна ожидаемо `failed`. Инварианты `accepted_jobs_reach_terminal_state`, `duplicate_effect_is_unique` и `old_pipeline_cannot_replace_new` получили `true`. Прогон занял 6981 ms. + +Это не exactly-once доставка: Redis может доставить одно сообщение несколько раз, а dispatcher может повторить `XADD`. Exactly-once относится только к наблюдаемому эффекту, который достигается идемпотентными ключами и compare-and-set в PostgreSQL. + +## Мониторинг и CI + +Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard разделён на блоки API, PostgreSQL, Valkey/Redis, object storage и attachment pipeline. В нём показаны timing основных ручек, CPU, соединения и транзакции PostgreSQL, cache hit/miss, память, доступность worker/dispatcher, throughput, p95 обработки, in-flight jobs, retry/reclaim, outbox backlog, Redis Streams lag, PEL, DLQ и idle consumers. + +CI для pull request и `main` выполняет contract check, OpenAPI validation, formatting, lint, race tests, build, публичный smoke, интеграционные тесты, persistence/failover lab4 и fault injection lab5. Production workflow запускается вручную и требует approval environment перед rolling deployment. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда | Результат | +| --- | --- | --- | +| Публичный API lab5 | `make test` | PASS: queued → ready, original и thumbnail проверены | +| PostgreSQL и Valkey | `docker compose --profile tests run --rm --build tests` | оба пакета PASS | +| Transform | `go test ./internal/messenger/attachment/transform` | thumbnail/crop/WebP, EXIF, alpha и лимиты PASS | +| Redis и outbox | `go test ./internal/broker/redisstream ./internal/outbox` | envelope, poison, owner fencing и lost lease PASS | +| Worker остановлен, outbox gap, restart broker/API | `make test-lab5-acceptance` | queued link и 409 до worker; после restart `ready`, original сохранён | +| Трек 5 | `make test-lab5` | 5/5 jobs terminal, 4 ready, 1 expected failed, все три инварианта true | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Ограничения -## Эксперименты и отказы +Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Decode и собственные циклы обработки прерываются по deadline; библиотечный вызов WebP обрабатывает ограниченный вариант до `1280×1280` и не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Источники и библиотеки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Redis Streams и consumer groups: . +- Redis persistence: . +- PostgreSQL `SKIP LOCKED`: . +- MinIO Go SDK: . +- OpenTelemetry trace context: . +- Go-пакеты: `pgx/v5`, `go-redis/v9`, `minio-go/v7`, `deepteams/webp`, Prometheus client. diff --git a/cmd/lab4experiment/main.go b/cmd/lab4experiment/main.go deleted file mode 100644 index 9ce5d5a..0000000 --- a/cmd/lab4experiment/main.go +++ /dev/null @@ -1,676 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "flag" - "fmt" - "image" - "image/jpeg" - "io" - "net/http" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" - - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" -) - -const experimentPassword = "Lab4-experiment-password" - -type apiClient struct { - baseURL string - client *http.Client - token string -} - -type attachment struct { - ID string `json:"id"` -} - -type setup struct { - client *apiClient - chatID string -} - -func main() { - if err := run(os.Args[1:], os.Stdout); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func run(arguments []string, output io.Writer) error { - if len(arguments) == 0 { - return errors.New("usage: lab4experiment memory|consistency [flags]") - } - switch arguments[0] { - case "memory": - return runMemory(arguments[1:], output) - case "consistency": - return runConsistency(arguments[1:], output) - default: - return errors.New("usage: lab4experiment memory|consistency [flags]") - } -} - -type memoryResult struct { - Replicas int `json:"replicas"` - Concurrency int `json:"concurrency"` - Rounds int `json:"rounds"` - SettleTimeMillis int64 `json:"settle_time_ms"` - ImageBytes int `json:"image_bytes"` - UploadRateBytes int64 `json:"upload_rate_bytes_per_second"` - SuccessfulUploads int64 `json:"successful_uploads"` - FailedUploads int64 `json:"failed_uploads"` - DurationMillis int64 `json:"duration_ms"` - HeapBaselineBytes int64 `json:"heap_baseline_bytes"` - HeapPeakBytes int64 `json:"heap_peak_bytes"` - HeapDeltaBytes int64 `json:"heap_delta_bytes"` - RSSBaselineBytes int64 `json:"rss_baseline_bytes"` - RSSPeakBytes int64 `json:"rss_peak_bytes"` - RSSDeltaBytes int64 `json:"rss_delta_bytes"` -} - -func runMemory(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("memory", flag.ContinueOnError) - flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") - prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL") - replicas := flags.Int("replicas", 2, "expected API replicas") - concurrency := flags.Int("concurrency", 12, "number of concurrent uploads") - rounds := flags.Int("rounds", 4, "number of upload waves") - width := flags.Int("width", 2048, "image width") - height := flags.Int("height", 2048, "image height") - uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload") - settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("invalid memory experiment flags") - } - if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 { - return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive") - } - - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - fixture, err := jpegFixture(*width, *height) - if err != nil { - return err - } - if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil { - return err - } - settle := time.NewTimer(*settleTime) - defer settle.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-settle.C: - } - heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) - if err != nil { - return err - } - rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes")) - if err != nil { - return err - } - environment, err := prepare(ctx, *baseURL) - if err != nil { - return err - } - - var successful atomic.Int64 - var failed atomic.Int64 - var firstError struct { - sync.Mutex - err error - } - heapPeak := heapBaseline - rssPeak := rssBaseline - loadDone := make(chan struct{}) - sampleDone := make(chan error, 1) - go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone) - started := time.Now() - for round := range *rounds { - var workers sync.WaitGroup - workers.Add(*concurrency) - for index := range *concurrency { - go func(round, index int) { - defer workers.Done() - reader := newRateReader(fixture, *uploadRate) - filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index) - if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil { - failed.Add(1) - firstError.Lock() - if firstError.err == nil { - firstError.err = err - } - firstError.Unlock() - return - } - successful.Add(1) - }(round, index) - } - workers.Wait() - } - close(loadDone) - if err := <-sampleDone; err != nil { - return err - } - result := memoryResult{ - Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(), - ImageBytes: len(fixture), UploadRateBytes: *uploadRate, - SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(), - HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline, - RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline, - } - if result.FailedUploads > 0 { - return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err) - } - return writeJSON(output, result) -} - -func sampleMemory( - ctx context.Context, - prometheusURL string, - expectedReplicas int, - done <-chan struct{}, - heapPeak *int64, - rssPeak *int64, - result chan<- error, -) { - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - result <- ctx.Err() - return - case <-done: - result <- nil - return - case <-ticker.C: - replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) - if err != nil { - result <- err - return - } - if replicas != int64(expectedReplicas) { - result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas) - return - } - heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) - if err != nil { - result <- err - return - } - rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes")) - if err != nil { - result <- err - return - } - *heapPeak = max(*heapPeak, heap) - *rssPeak = max(*rssPeak, rss) - } - } -} - -type consistencyResult struct { - PageSize int `json:"page_size"` - SeededConsistent int `json:"seeded_consistent"` - FreshApply reconciliationRun `json:"fresh_apply"` - DryRun reconciliationRun `json:"dry_run"` - Apply reconciliationRun `json:"apply"` - RepeatedApply reconciliationRun `json:"repeated_apply"` - FreshPreserved bool `json:"fresh_preserved"` - OrphanDeleted bool `json:"orphan_deleted"` - UnusedDeleted bool `json:"unused_metadata_deleted"` - InUsePreserved bool `json:"in_use_metadata_preserved"` - AbandonedDeleted bool `json:"abandoned_attachment_deleted"` - TotalDurationMillis int64 `json:"total_duration_ms"` -} - -type reconciliationRun struct { - DurationMillis int64 `json:"duration_ms"` - Report reconcile.Report `json:"report"` -} - -func runConsistency(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("consistency", flag.ContinueOnError) - flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") - pageSize := flags.Int("page-size", 10, "reconciliation page size") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("invalid consistency experiment flags") - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - settings, err := config.LoadReconciliation() - if err != nil { - return err - } - pool, err := database.Open(ctx, database.Config{ - URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, - MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, - HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - }) - if err != nil { - return err - } - defer pool.Close() - objects, err := objectstore.New(objectstore.Config{ - Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, - AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, - }) - if err != nil { - return err - } - metadata := postgres.New(pool) - clean, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour, - }) - if err != nil { - return err - } - initial, err := clean.Run(ctx) - if err != nil { - return err - } - if hasAnomalies(initial) { - return errors.New("consistency experiment requires a stack without existing attachment anomalies") - } - - started := time.Now() - environment, err := prepare(ctx, *baseURL) - if err != nil { - return err - } - fixture, err := jpegFixture(320, 200) - if err != nil { - return err - } - seededConsistent := *pageSize + 1 - for index := range seededConsistent { - filename := fmt.Sprintf("consistent-%03d.jpg", index) - if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil { - return err - } - } - unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil { - return err - } - abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil { - return fmt.Errorf("prepare abandoned pending attachment: %w", err) - } - keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID}) - if err != nil { - return err - } - if err := objects.Delete(ctx, keys[unused.ID]); err != nil { - return err - } - if err := objects.Delete(ctx, keys[inUse.ID]); err != nil { - return err - } - orphanKey := "attachments/experiment-" + uuid.NewString() + "/original" - if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil { - return err - } - - fresh, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, - UnreferencedRetention: time.Hour, Apply: true, - }) - if err != nil { - return err - } - freshRun, err := measuredReconciliation(ctx, fresh) - if err != nil { - return err - } - freshPreserved := objectExists(ctx, objects, orphanKey) && - attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) && - attachmentExists(ctx, pool, abandoned.ID) - if _, err := pool.Exec(ctx, ` - UPDATE attachments - SET created_at = clock_timestamp() - INTERVAL '2 hours' - WHERE id = $1`, abandoned.ID); err != nil { - return fmt.Errorf("age abandoned attachment: %w", err) - } - - dryRun, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, - }) - if err != nil { - return err - } - dryRunResult, err := measuredReconciliation(ctx, dryRun) - if err != nil { - return err - } - apply, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true, - }) - if err != nil { - return err - } - applyResult, err := measuredReconciliation(ctx, apply) - if err != nil { - return err - } - repeatedResult, err := measuredReconciliation(ctx, apply) - if err != nil { - return err - } - - result := consistencyResult{ - PageSize: *pageSize, SeededConsistent: seededConsistent, - FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult, - FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey), - UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID), - AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), - TotalDurationMillis: time.Since(started).Milliseconds(), - } - if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted || - repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 { - return fmt.Errorf("unexpected reconciliation result: %+v", result) - } - return writeJSON(output, result) -} - -func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) { - started := time.Now() - report, err := reconciler.Run(ctx) - return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err -} - -func hasAnomalies(report reconcile.Report) bool { - return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 || - report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 || - report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 || - report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0 -} - -func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) { - rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids) - if err != nil { - return nil, fmt.Errorf("query attachment keys: %w", err) - } - defer rows.Close() - keys := make(map[string]string, len(ids)) - for rows.Next() { - var id string - var key string - if err := rows.Scan(&id, &key); err != nil { - return nil, fmt.Errorf("scan attachment key: %w", err) - } - keys[id] = key - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate attachment keys: %w", err) - } - if len(keys) != len(ids) { - return nil, errors.New("not all attachment keys were found") - } - return keys, nil -} - -func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool { - var exists bool - return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists -} - -func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool { - _, err := objects.Stat(ctx, key) - return err == nil -} - -func prepare(ctx context.Context, baseURL string) (setup, error) { - client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}} - username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "") - var user struct { - ID string `json:"id"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{ - "username": username, "display_name": "Lab 4 experiment", "password": experimentPassword, - }, &user, ""); err != nil { - return setup{}, err - } - credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword)) - var session struct { - Token string `json:"token"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil { - return setup{}, err - } - client.token = session.Token - var chat struct { - ID string `json:"id"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{ - "title": "Lab 4 experiment", "member_ids": []string{user.ID}, - }, &chat, "Bearer "+client.token); err != nil { - return setup{}, err - } - return setup{client: client, chatID: chat.ID}, nil -} - -func (c *apiClient) upload( - ctx context.Context, - chatID string, - filename string, - body io.Reader, - contentLength int64, -) (attachment, error) { - path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename - request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body) - if err != nil { - return attachment{}, err - } - request.ContentLength = contentLength - request.Header.Set("Authorization", "Bearer "+c.token) - request.Header.Set("Content-Type", "image/jpeg") - response, err := c.client.Do(request) - if err != nil { - return attachment{}, err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusCreated { - message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message) - } - var result attachment - if err := json.NewDecoder(response.Body).Decode(&result); err != nil { - return attachment{}, err - } - return result, nil -} - -func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error { - return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{ - "text": "", "attachment_ids": []string{attachmentID}, - }, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString()) -} - -func (c *apiClient) jsonRequest( - ctx context.Context, - method string, - path string, - payload any, - result any, - authorization string, - extraHeader ...string, -) error { - var body io.Reader - if payload != nil { - encoded, err := json.Marshal(payload) - if err != nil { - return err - } - body = bytes.NewReader(encoded) - } - request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) - if err != nil { - return err - } - if payload != nil { - request.Header.Set("Content-Type", "application/json") - } - if authorization != "" { - request.Header.Set("Authorization", authorization) - } - for index := 0; index+1 < len(extraHeader); index += 2 { - request.Header.Set(extraHeader[index], extraHeader[index+1]) - } - response, err := c.client.Do(request) - if err != nil { - return err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode < 200 || response.StatusCode >= 300 { - message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message) - } - if result == nil { - return nil - } - return json.NewDecoder(response.Body).Decode(result) -} - -func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) - if err == nil && count == int64(expected) { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err()) - case <-ticker.C: - } - } -} - -func activeReplicasQuery() string { - return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))` -} - -func freshMetricQuery(metric string) string { - return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric) -} - -func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil) - if err != nil { - return 0, err - } - values := request.URL.Query() - values.Set("query", query) - request.URL.RawQuery = values.Encode() - response, err := http.DefaultClient.Do(request) - if err != nil { - return 0, err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusOK { - return 0, fmt.Errorf("prometheus status %d", response.StatusCode) - } - var payload struct { - Status string `json:"status"` - Data struct { - Result []struct { - Value []json.RawMessage `json:"value"` - } `json:"result"` - } `json:"data"` - } - if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { - return 0, err - } - if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 { - return 0, errors.New("prometheus query returned no scalar value") - } - var raw string - if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil { - return 0, err - } - value, err := strconv.ParseFloat(raw, 64) - return int64(value), err -} - -func jpegFixture(width, height int) ([]byte, error) { - pixels := image.NewRGBA(image.Rect(0, 0, width, height)) - state := uint64(0x9e3779b97f4a7c15) - for index := range pixels.Pix { - state ^= state << 13 - state ^= state >> 7 - state ^= state << 17 - pixels.Pix[index] = byte(state >> 24) - } - var encoded bytes.Buffer - if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil { - return nil, err - } - return encoded.Bytes(), nil -} - -type rateReader struct { - reader *bytes.Reader - rate int64 - started time.Time - read int64 -} - -func newRateReader(data []byte, rate int64) *rateReader { - return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()} -} - -func (r *rateReader) Read(buffer []byte) (int, error) { - for { - allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read - if allowed > 0 { - if int64(len(buffer)) > allowed { - buffer = buffer[:allowed] - } - count, err := r.reader.Read(buffer) - r.read += int64(count) - return count, err - } - time.Sleep(10 * time.Millisecond) - } -} - -func writeJSON(output io.Writer, value any) error { - encoder := json.NewEncoder(output) - encoder.SetIndent("", " ") - return encoder.Encode(value) -} diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go new file mode 100644 index 0000000..5dc3072 --- /dev/null +++ b/cmd/lab5experiment/api_scenario.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + _ "image/jpeg" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type apiState struct { + BaseURL string `json:"base_url"` + Token string `json:"token"` + AttachmentID string `json:"attachment_id"` +} + +type apiUser struct { + ID string `json:"id"` +} + +type apiSession struct { + Token string `json:"token"` +} + +type apiChat struct { + ID string `json:"id"` +} + +func runAPIProduce(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("api-produce", flag.ContinueOnError) + flags.SetOutput(io.Discard) + baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-produce [-base-url URL]") + } + client := &http.Client{Timeout: 10 * time.Second} + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + password := "Lab5-" + uuid.NewString() + owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password) + if err != nil { + return err + } + token, err := createAPISession(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + var attachment model.Attachment + response, err := apiRequest( + client, http.MethodPost, + *baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png", + token, "image/png", fixture, &attachment, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued { + return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status) + } + messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}}) + if err != nil { + return err + } + response, err = apiRequestWithHeaders( + client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages", + token, "application/json", messageBody, nil, + map[string]string{"Idempotency-Key": uuid.NewString()}, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("link queued attachment returned %d", response.StatusCode) + } + response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil) + if err != nil { + return err + } + if response.StatusCode != http.StatusConflict { + return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode) + } + return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID}) +} + +func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { + flags := flag.NewFlagSet("api-verify", flag.ContinueOnError) + flags.SetOutput(io.Discard) + waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]") + } + var state apiState + if err := json.NewDecoder(input).Decode(&state); err != nil { + return fmt.Errorf("decode API scenario state: %w", err) + } + client := &http.Client{Timeout: 10 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout) + defer cancel() + var attachment model.Attachment + for { + response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("attachment metadata returned %d", response.StatusCode) + } + if attachment.Status == model.AttachmentReady { + break + } + if attachment.Status == model.AttachmentFailed { + return fmt.Errorf("attachment failed with %v", attachment.ErrorCode) + } + if err := wait(ctx, 250*time.Millisecond); err != nil { + return err + } + } + if len(attachment.Variants) != 3 { + return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants)) + } + response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) { + return errors.New("original changed after worker and broker restart") + } + response, thumbnail, err := apiBytes( + client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" { + return errors.New("thumbnail response is invalid") + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail)) + if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 { + return errors.New("thumbnail dimensions are invalid") + } + return json.NewEncoder(output).Encode(map[string]any{ + "accepted_while_worker_stopped": true, + "accepted_before_outbox_publish": true, + "linked_while_queued": true, + "content_conflict_while_queued": true, + "ready_after_api_broker_restart": true, + "original_preserved": true, + "thumbnail_dimensions": "256x160", + }) +} + +func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) { + body, err := json.Marshal(map[string]string{ + "username": username, "display_name": username, "password": password, + }) + if err != nil { + return apiUser{}, err + } + var user apiUser + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user) + if err != nil { + return apiUser{}, err + } + if response.StatusCode != http.StatusCreated { + return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode) + } + return user, nil +} + +func createAPISession(client *http.Client, baseURL, username, password string) (string, error) { + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Basic "+credentials) + response, err := client.Do(request) + if err != nil { + return "", err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create session returned %d", response.StatusCode) + } + var session apiSession + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return "", err + } + return session.Token, nil +} + +func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) { + body, err := json.Marshal(map[string]any{ + "title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID}, + }) + if err != nil { + return apiChat{}, err + } + var chat apiChat + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat) + if err != nil { + return apiChat{}, err + } + if response.StatusCode != http.StatusCreated { + return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode) + } + return chat, nil +} + +func apiRequest( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, +) (*http.Response, error) { + return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil) +} + +func apiRequestWithHeaders( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, + headers map[string]string, +) (*http.Response, error) { + request, err := http.NewRequest(method, requestURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + for name, value := range headers { + request.Header.Set(name, value) + } + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if result != nil && response.StatusCode < http.StatusBadRequest { + if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil { + return nil, err + } + } + return response, nil +} + +func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) { + request, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, nil, err + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + if err != nil { + return nil, nil, err + } + defer func() { _ = response.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + return response, body, err +} diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go new file mode 100644 index 0000000..3749235 --- /dev/null +++ b/cmd/lab5experiment/main.go @@ -0,0 +1,607 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "os" + "strings" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +const ( + experimentLease = 2 * time.Second + maxAttempts = 5 +) + +type result struct { + StartedAt string `json:"started_at"` + DurationMillis int64 `json:"duration_ms"` + PipelineVersion int `json:"pipeline_version"` + FaultScenarios []scenario `json:"fault_scenarios"` + DuplicateDeliveries int `json:"duplicate_deliveries"` + V1ObjectCount int `json:"v1_object_count"` + V2ObjectCount int `json:"v2_object_count"` + AcceptedJobs int `json:"accepted_jobs"` + TerminalJobs int `json:"terminal_jobs"` + ReadyJobs int `json:"ready_jobs"` + FailedJobs int `json:"failed_jobs"` + PermanentErrorCode string `json:"permanent_error_code"` + ReplayAttempt int `json:"replay_attempt"` + InvariantResults map[string]bool `json:"invariant_results"` +} + +type scenario struct { + Name string `json:"name"` + AttemptAfterRecovery int `json:"attempt_after_recovery"` + ObjectsBeforeRecovery int `json:"objects_before_recovery"` + ObjectsAfterRecovery int `json:"objects_after_recovery"` + FinalStatus string `json:"final_status"` +} + +type experiment struct { + settings config.WorkerConfig + repository *postgres.Store + objects *objectstore.Store + broker *redisstream.Broker + dispatcher *outbox.Dispatcher + worker *worker.Worker + ownerID string + chatID string + fixture []byte +} + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(arguments []string, output io.Writer) error { + if len(arguments) > 0 { + switch arguments[0] { + case "api-produce": + return runAPIProduce(arguments[1:], output) + case "api-verify": + return runAPIVerify(arguments[1:], os.Stdin, output) + } + } + flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError) + flags.SetOutput(io.Discard) + timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 { + return errors.New("usage: lab5experiment [-timeout 2m]") + } + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + experiment, closeExperiment, err := newExperiment(ctx) + if err != nil { + return err + } + defer closeExperiment() + + report, err := experiment.run(ctx) + if err != nil { + return err + } + report.StartedAt = started.UTC().Format(time.RFC3339) + report.DurationMillis = time.Since(started).Milliseconds() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func newExperiment(ctx context.Context) (*experiment, func(), error) { + settings, err := config.LoadWorker() + if err != nil { + return nil, nil, err + } + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return nil, nil, err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + pool.Close() + return nil, nil, err + } + redisOptions, err := redis.ParseURL(settings.BrokerURL) + if err != nil { + pool.Close() + return nil, nil, err + } + client := redis.NewClient(redisOptions) + if err := client.Ping(ctx).Err(); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + consumer := "lab5experiment-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second, + ClaimMinIdle: experimentLease, ReadCount: 32, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + if err := broker.Ensure(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + repository := postgres.New(pool) + dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ + BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, + RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processorWorker, err := worker.New( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, + worker.Config{ + WorkerID: consumer, Concurrency: 1, + LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond, + ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + fixture, err := pngFixture(320, 200) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + value := &experiment{ + settings: settings, repository: repository, objects: objects, broker: broker, + dispatcher: dispatcher, worker: processorWorker, fixture: fixture, + } + if err := value.seed(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + closeExperiment := func() { + pool.Close() + _ = client.Close() + } + return value, closeExperiment, nil +} + +func (e *experiment) seed(ctx context.Context) error { + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment") + if err != nil { + return err + } + member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment") + if err != nil { + return err + } + chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID) + if err != nil { + return err + } + e.ownerID = owner.ID + e.chatID = chat.ID + return nil +} + +func (e *experiment) run(ctx context.Context) (result, error) { + report := result{ + PipelineVersion: 1, + InvariantResults: map[string]bool{ + "accepted_jobs_reach_terminal_state": false, + "duplicate_effect_is_unique": false, + "old_pipeline_cannot_replace_new": false, + }, + } + for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} { + observed, err := e.runLeaseRecovery(ctx, fault) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, observed) + report.AcceptedJobs++ + if observed.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + } + permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx) + if err != nil { + return result{}, err + } + report.AcceptedJobs++ + report.TerminalJobs++ + report.FailedJobs++ + report.PermanentErrorCode = permanentCode + report.ReplayAttempt = replayAttempt + + commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, commitScenario) + report.AcceptedJobs++ + if commitScenario.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + + payload, err := redisstream.EncodeEnvelope(envelope) + if err != nil { + return result{}, err + } + for range 2 { + if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil { + return result{}, err + } + } + duplicateDeliveries, err := e.readDeliveries(ctx, 2) + if err != nil { + return result{}, err + } + for _, delivery := range duplicateDeliveries { + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return result{}, err + } + } + report.DuplicateDeliveries = len(duplicateDeliveries) + report.V1ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V1ObjectCount != 4 { + return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount) + } + report.InvariantResults["duplicate_effect_is_unique"] = true + + oldPayload := payload + v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1} + if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil { + return result{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return result{}, err + } + v2Delivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil { + return result{}, err + } + if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil { + return result{}, err + } + staleDelivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil { + return result{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return result{}, err + } + if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 { + return result{}, errors.New("new pipeline version was not preserved after stale delivery") + } + report.V2ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V2ObjectCount != 8 { + return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount) + } + report.InvariantResults["old_pipeline_cannot_replace_new"] = true + report.AcceptedJobs++ + report.ReadyJobs++ + report.TerminalJobs++ + report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs + return report, nil +} + +func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) { + attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png") + if err != nil { + return "", 0, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil { + return "", 0, errors.New("corrupted staging object did not reach failed") + } + firstCode := *metadata.ErrorCode + replayed, err := e.repository.ReplayFailedAttachment( + ctx, attachmentID, "lab5experiment", "verify explicit replay fencing", + ) + if err != nil { + return "", 0, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", 0, err + } + replayDelivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode { + return "", 0, errors.New("replayed permanent failure changed terminal classification") + } + return firstCode, replayed.Attempt, nil +} + +func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, fmt.Errorf("fault %s returned %v", fault, err) + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + if fault == worker.FaultBeforeSideEffect && before != 0 { + return scenario{}, errors.New("objects were written before the injected side effect") + } + if fault == worker.FaultAfterSideEffect && before != 4 { + return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before) + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, err + } + recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now()) + if err != nil { + return scenario{}, err + } + if recovered.Requeued != 1 { + return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued) + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return scenario{}, err + } + retry, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil { + return scenario{}, err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String()) + if err != nil { + return scenario{}, err + } + return scenario{ + Name: string(fault), AttemptAfterRecovery: job.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, nil +} + +func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err) + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + return scenario{ + Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, delivery.Envelope, attachmentID, nil +} + +func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) { + attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png") + return attachmentID, job, err +} + +func (e *experiment) queueAttachmentData( + ctx context.Context, + data []byte, + contentType string, +) (string, string, model.AttachmentJob, error) { + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/experiment" + digest := sha256.Sum256(data) + _, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png", + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1, + }) + if err != nil { + return "", "", model.AttachmentJob{}, err + } + if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil { + return "", "", model.AttachmentJob{}, err + } + job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1} + if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil { + return "", "", model.AttachmentJob{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", "", model.AttachmentJob{}, err + } + return attachmentID, stagingKey, job, nil +} + +func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) { + deliveries, err := e.readDeliveries(ctx, 1) + if err != nil { + return redisstream.Delivery{}, err + } + return deliveries[0], nil +} + +func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) { + var result []redisstream.Delivery + for len(result) < expected { + deliveries, err := e.broker.Read(ctx) + if err != nil { + return nil, err + } + result = append(result, deliveries...) + } + if len(result) != expected { + return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected) + } + return result, nil +} + +func (e *experiment) ackReclaimed(ctx context.Context) error { + deliveries, _, err := e.broker.Reclaim(ctx, "0-0") + if err != nil { + return err + } + if len(deliveries) != 1 { + return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries)) + } + return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone) +} + +func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) { + objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100) + if err != nil { + return 0, err + } + if next != "" { + return 0, errors.New("unexpected object pagination") + } + return len(objects), nil +} + +func pngFixture(width, height int) ([]byte, error) { + value := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } + var output bytes.Buffer + if err := png.Encode(&output, value); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func wait(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index c6ab63e..80230b8 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -24,8 +24,17 @@ func run(logger *slog.Logger, arguments []string) error { if len(arguments) == 1 && arguments[0] == "migrate" { return bootstrap.RunMigrations() } + if len(arguments) == 1 && arguments[0] == "dispatch" { + return bootstrap.RunDispatcher(logger) + } + if len(arguments) == 1 && arguments[0] == "worker" { + return bootstrap.RunWorker(logger) + } if arguments[0] == "reconcile-attachments" { return bootstrap.RunReconciliation(arguments[1:], os.Stdout) } - return errors.New("usage: messenger [migrate|reconcile-attachments]") + if arguments[0] == "replay-attachment" { + return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout) + } + return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]") } diff --git a/compose.yaml b/compose.yaml index f1c370a..291eaec 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,9 @@ +x-messenger-config-volume: &messenger-config-volume + type: bind + source: ./config/messenger.yaml + target: /etc/messenger/messenger.yaml + read_only: true + services: postgres: image: postgres:17.6-alpine3.22 @@ -34,6 +40,32 @@ services: restart: unless-stopped networks: [data] + redis-broker: + image: redis:8.2.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - always + - --maxmemory-policy + - noeviction + - --requirepass + - ${REDIS_BROKER_PASSWORD} + environment: + REDIS_BROKER_PASSWORD: ${REDIS_BROKER_PASSWORD} + volumes: + - redis_broker_data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_BROKER_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${REDIS_BROKER_CPUS} + mem_limit: ${REDIS_BROKER_MEMORY} + restart: unless-stopped + networks: [data] + s3: build: context: . @@ -89,14 +121,10 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "migrate"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} - MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS} + volumes: + - *messenger-config-volume depends_on: postgres: condition: service_healthy @@ -111,42 +139,19 @@ services: build: . image: ${API_IMAGE:-mtusi-messenger:lab5} environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml APP_HOST: ${APP_HOST} APP_PORT: ${APP_PORT} DATABASE_URL: ${DATABASE_URL} CACHE_URL: ${CACHE_URL} - SESSION_STORE: ${SESSION_STORE} - SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} - IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} - REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS} - HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS} - HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS} - HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS} - HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS} - HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS} - PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB} - PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS} - PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM} - PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} - PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} SWAGGER_PORT: ${SWAGGER_PORT} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES} - ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES} + volumes: + - *messenger-config-volume expose: - "8080" depends_on: @@ -170,6 +175,66 @@ services: restart: unless-stopped networks: [application, data] + dispatcher: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/messenger", "dispatch"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${DISPATCHER_CPUS} + mem_limit: ${DISPATCHER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + worker: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/messenger", "worker"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${WORKER_CPUS} + mem_limit: ${WORKER_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [data, monitoring] + nginx: build: context: . @@ -222,6 +287,32 @@ services: mem_limit: ${TEST_MEMORY} networks: [data] + lab5-experiment: + profiles: ["experiments"] + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["/app/lab5experiment"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [application, data] + postgres-exporter: image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 environment: @@ -229,6 +320,9 @@ services: DATA_SOURCE_USER: ${POSTGRES_USER} DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} PG_EXPORTER_COLLECTION_TIMEOUT: 5s + PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres_exporter/queries.yml + volumes: + - ./monitoring/postgres-queries.yml:/etc/postgres_exporter/queries.yml:ro depends_on: postgres: condition: service_healthy @@ -251,6 +345,21 @@ services: restart: unless-stopped networks: [data, monitoring] + redis-broker-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://redis-broker:6379 + REDIS_USER: default + REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD} + REDIS_EXPORTER_CHECK_SINGLE_STREAMS: attachment-jobs,attachment-jobs-failed + depends_on: + redis-broker: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + nginx-exporter: image: nginx/nginx-prometheus-exporter:1.5.1 command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] @@ -285,6 +394,9 @@ services: - nginx-exporter - postgres-exporter - valkey-exporter + - redis-broker-exporter + - dispatcher + - worker - s3 cpus: ${PROMETHEUS_CPUS} mem_limit: ${PROMETHEUS_MEMORY} @@ -314,6 +426,7 @@ services: volumes: postgres_data: valkey_data: + redis_broker_data: prometheus_data: grafana_data: s3_data: diff --git a/config/messenger.yaml b/config/messenger.yaml new file mode 100644 index 0000000..031e152 --- /dev/null +++ b/config/messenger.yaml @@ -0,0 +1,76 @@ +version: 1 + +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 + +database: + min_connections: 1 + max_connections: 10 + max_connection_age: 30m + max_connection_idle: 5m + health_check_period: 30s + dependency_timeout: 5s + +migration: + timeout: 30s + +object_storage: + use_tls: false + timeout: 5s + +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 + +pipeline: + version: 1 + +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 + +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 + +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 + +monitoring: + metrics_address: 0.0.0.0:9091 diff --git a/deploy/compose.images.yaml b/deploy/compose.images.yaml index 4251b21..7d98319 100644 --- a/deploy/compose.images.yaml +++ b/deploy/compose.images.yaml @@ -5,6 +5,12 @@ services: migrate: image: ${API_IMAGE:?API_IMAGE is required} + dispatcher: + image: ${API_IMAGE:?API_IMAGE is required} + + worker: + image: ${API_IMAGE:?API_IMAGE is required} + nginx: image: ${NGINX_IMAGE:?NGINX_IMAGE is required} diff --git a/evidence/lab5-idempotency.json b/evidence/lab5-idempotency.json new file mode 100644 index 0000000..c6e9050 --- /dev/null +++ b/evidence/lab5-idempotency.json @@ -0,0 +1,42 @@ +{ + "started_at": "2026-10-22T14:19:25Z", + "duration_ms": 6981, + "pipeline_version": 1, + "fault_scenarios": [ + { + "name": "before_side_effect", + "attempt_after_recovery": 2, + "objects_before_recovery": 0, + "objects_after_recovery": 4, + "final_status": "ready" + }, + { + "name": "after_side_effect", + "attempt_after_recovery": 2, + "objects_before_recovery": 4, + "objects_after_recovery": 4, + "final_status": "ready" + }, + { + "name": "after_commit", + "attempt_after_recovery": 1, + "objects_before_recovery": 4, + "objects_after_recovery": 4, + "final_status": "ready" + } + ], + "duplicate_deliveries": 2, + "v1_object_count": 4, + "v2_object_count": 8, + "accepted_jobs": 5, + "terminal_jobs": 5, + "ready_jobs": 4, + "failed_jobs": 1, + "permanent_error_code": "invalid_image", + "replay_attempt": 2, + "invariant_results": { + "accepted_jobs_reach_terminal_state": true, + "duplicate_effect_is_unique": true, + "old_pipeline_cannot_replace_new": true + } +} diff --git a/evidence/lab5-resilience.json b/evidence/lab5-resilience.json new file mode 100644 index 0000000..c82d186 --- /dev/null +++ b/evidence/lab5-resilience.json @@ -0,0 +1 @@ +{"accepted_before_outbox_publish":true,"accepted_while_worker_stopped":true,"content_conflict_while_queued":true,"linked_while_queued":true,"original_preserved":true,"ready_after_api_broker_restart":true,"thumbnail_dimensions":"256x160"} diff --git a/go.mod b/go.mod index 10f19e4..563b37a 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module gitea.privcloud.tech/dizi/mtusi2026lab5 go 1.26.0 require ( + github.com/deepteams/webp v1.2.8 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 github.com/minio/minio-go/v7 v7.0.99 github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 @@ -18,6 +20,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect diff --git a/go.sum b/go.sum index 5aa456c..edfd0ad 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA= +github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -64,6 +72,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go new file mode 100644 index 0000000..97ce3ec --- /dev/null +++ b/internal/bootstrap/pipeline.go @@ -0,0 +1,201 @@ +package bootstrap + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +func RunDispatcher(logger *slog.Logger) error { + settings, err := config.LoadDispatcher() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher", + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + metrics := pipeline.NewMetrics() + dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ + BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, + PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, + RetryDelay: settings.RetryDelay, Retention: settings.Retention, + CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize, + }) + if err != nil { + return err + } + logger.Info("outbox dispatcher started", "stream", settings.Stream) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run) +} + +func RunWorker(logger *slog.Logger) error { + settings, err := config.LoadWorker() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), workerDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + hostname, err := os.Hostname() + if err != nil { + return err + } + consumer := hostname + "-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + return err + } + metrics := pipeline.NewMetrics() + processorWorker, err := worker.New( + logger, postgres.New(pool), broker, objects, processor, metrics, + worker.Config{ + WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration, + HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout, + ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, + RetryBase: settings.RetryBase, RetryMax: settings.RetryMax, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + return err + } + logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run) +} + +func newRedisClient(rawURL string) (*redis.Client, error) { + options, err := redis.ParseURL(rawURL) + if err != nil { + return nil, err + } + client := redis.NewClient(options) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, err + } + return client, nil +} + +func runPipelineProcess( + logger *slog.Logger, + metricsAddress string, + metricsHandler http.Handler, + run func(context.Context) error, +) error { + mux := http.NewServeMux() + mux.Handle("/metrics", metricsHandler) + mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + listener, err := net.Listen("tcp", metricsAddress) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + runResult := make(chan error, 1) + go func() { runResult <- run(ctx) }() + + var result error + select { + case result = <-runResult: + stop() + case result = <-serveResult: + stop() + case <-ctx.Done(): + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil && result == nil { + result = err + } + if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) { + return nil + } + logger.Info("pipeline process stopped") + return result +} + +func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func workerDatabaseConfig(settings config.WorkerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go index 359ddbd..447dd23 100644 --- a/internal/bootstrap/reconciliation.go +++ b/internal/bootstrap/reconciliation.go @@ -64,6 +64,7 @@ func RunReconciliation(arguments []string, output io.Writer) error { } reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, + OrphanPrefixes: []string{"staging/"}, UnreferencedRetention: *retention, Apply: *apply, }) if err != nil { diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go new file mode 100644 index 0000000..56b9f2e --- /dev/null +++ b/internal/bootstrap/replay.go @@ -0,0 +1,39 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "io" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +func RunAttachmentReplay(arguments []string, output io.Writer) error { + if len(arguments) != 3 { + return errors.New("usage: messenger replay-attachment ") + } + settings, err := config.LoadJobAdmin() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + job, err := postgres.New(pool).ReplayFailedAttachment( + context.Background(), arguments[0], arguments[1], arguments[2], + ) + if err != nil { + return err + } + _, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt) + return err +} diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go index 7073922..121e328 100644 --- a/internal/bootstrap/server.go +++ b/internal/bootstrap/server.go @@ -54,7 +54,7 @@ func RunServer(logger *slog.Logger) error { attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{ MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory, - CleanupTimeout: settings.RequestTimeout, + CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion, }) if err != nil { return err diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go new file mode 100644 index 0000000..35d46f0 --- /dev/null +++ b/internal/broker/redisstream/broker.go @@ -0,0 +1,291 @@ +package redisstream + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" +) + +const ( + payloadField = "payload" + failedPayloadMaxBytes = 64 << 10 +) + +var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + +var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer") + +var _ outbox.Publisher = (*Broker)(nil) + +type Config struct { + Stream string + Group string + Consumer string + FailedStream string + FailedMaxLen int64 + Block time.Duration + ClaimMinIdle time.Duration + ReadCount int64 +} + +type Broker struct { + client redis.UniversalClient + config Config +} + +type Delivery struct { + ID string + Envelope Envelope + DecodeError error + + broker *Broker + payload []byte +} + +type FailedDelivery struct { + Code string + FailedAt time.Time +} + +func New(client redis.UniversalClient, config Config) (*Broker, error) { + if client == nil { + return nil, errors.New("redis client is required") + } + if err := config.validate(); err != nil { + return nil, err + } + return &Broker{client: client, config: config}, nil +} + +func (c Config) validate() error { + switch { + case c.Stream == "": + return errors.New("stream is required") + case c.Group == "": + return errors.New("consumer group is required") + case c.Consumer == "": + return errors.New("consumer name is required") + case c.FailedStream == "": + return errors.New("failed stream is required") + case c.FailedStream == c.Stream: + return errors.New("failed stream must differ from job stream") + case c.FailedMaxLen <= 0: + return errors.New("failed stream maximum length must be positive") + case c.Block < 0: + return errors.New("block duration must not be negative") + case c.ClaimMinIdle <= 0: + return errors.New("claim minimum idle time must be positive") + case c.ReadCount <= 0: + return errors.New("read count must be positive") + default: + return nil + } +} + +func (b *Broker) Ensure(ctx context.Context) error { + err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err() + if err != nil && !isBusyGroup(err) { + return fmt.Errorf("create redis stream consumer group: %w", err) + } + return nil +} + +func isBusyGroup(err error) bool { + return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP") +} + +func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) { + envelope, err := DecodeEnvelope(payload) + if err != nil { + return "", err + } + payload, err = EncodeEnvelope(envelope) + if err != nil { + return "", err + } + + id, err := b.client.XAdd(ctx, &redis.XAddArgs{ + Stream: b.config.Stream, + Values: map[string]any{payloadField: string(payload)}, + }).Result() + if err != nil { + return "", fmt.Errorf("publish redis stream job: %w", err) + } + return id, nil +} + +func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { + streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: b.config.Group, + Consumer: b.config.Consumer, + Streams: []string{b.config.Stream, ">"}, + Count: b.config.ReadCount, + Block: b.config.Block, + }).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read redis stream jobs: %w", err) + } + return b.deliveries(streams) +} + +func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) { + messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: b.config.Stream, + Group: b.config.Group, + Consumer: b.config.Consumer, + MinIdle: b.config.ClaimMinIdle, + Start: start, + Count: b.config.ReadCount, + }).Result() + if err != nil { + return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err) + } + + deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}}) + if err != nil { + return nil, start, err + } + return deliveries, next, nil +} + +func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) { + var deliveries []Delivery + for _, stream := range streams { + for _, message := range stream.Messages { + delivery := Delivery{ID: message.ID, broker: b} + payload, err := messagePayload(message) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.payload = boundedPayload(payload) + envelope, err := DecodeEnvelope(payload) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.Envelope = envelope + delivery.payload, err = EncodeEnvelope(envelope) + if err != nil { + return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err) + } + deliveries = append(deliveries, delivery) + } + } + return deliveries, nil +} + +func boundedPayload(payload []byte) []byte { + if len(payload) > failedPayloadMaxBytes { + payload = payload[:failedPayloadMaxBytes] + } + return append([]byte(nil), payload...) +} + +func messagePayload(message redis.XMessage) ([]byte, error) { + value, ok := message.Values[payloadField] + if !ok { + return nil, errors.New("payload field is missing") + } + switch payload := value.(type) { + case string: + return []byte(payload), nil + case []byte: + return payload, nil + default: + return nil, fmt.Errorf("payload has type %T", value) + } +} + +func (d Delivery) Ack(ctx context.Context) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, + ).Int64() + if err != nil { + return fmt.Errorf("ack redis stream job: %w", err) + } + if acked == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const ackOwnedScript = ` +local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[1], ARGV[2]) +end +return acked +` + +func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + if !failureCodePattern.MatchString(failure.Code) { + return errors.New("failure code is invalid") + } + if failure.FailedAt.IsZero() { + return errors.New("failure time is required") + } + + payload := d.payload + if d.DecodeError == nil && len(payload) == 0 { + var err error + payload, err = EncodeEnvelope(d.Envelope) + if err != nil { + return err + } + } + moved, err := d.broker.client.Eval(ctx, moveToFailedScript, + []string{d.broker.config.FailedStream, d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code, + failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt), + strconv.FormatInt(d.broker.config.FailedMaxLen, 10), + ).Int64() + if err != nil { + return fmt.Errorf("move redis stream job to failed stream: %w", err) + } + if moved == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const moveToFailedScript = ` +local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*', + 'payload', ARGV[4], + 'source_message_id', ARGV[2], + 'error_code', ARGV[5], + 'failed_at', ARGV[6], + 'attempt', ARGV[7]) +local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[2], ARGV[2]) +end +return acked +` diff --git a/internal/broker/redisstream/broker_test.go b/internal/broker/redisstream/broker_test.go new file mode 100644 index 0000000..5d96c42 --- /dev/null +++ b/internal/broker/redisstream/broker_test.go @@ -0,0 +1,93 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestBrokerDecodesDeliveries(t *testing.T) { + payload, err := EncodeEnvelope(validEnvelope()) + require.NoError(t, err) + + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: string(payload)}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Equal(t, "1-0", deliveries[0].ID) + require.Equal(t, validEnvelope(), deliveries[0].Envelope) +} + +func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) { + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: `{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"private/object" + }`}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Contains(t, string(deliveries[0].payload), `"storage_key":"private/object"`) +} + +func TestBrokerBoundsMalformedPayloadForFailedStream(t *testing.T) { + payload := make([]byte, failedPayloadMaxBytes+1) + for index := range payload { + payload[index] = 'x' + } + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: payload}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Len(t, deliveries[0].payload, failedPayloadMaxBytes) +} + +func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) { + broker := &Broker{} + _, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`)) + require.Error(t, err) +} + +func TestConfigValidation(t *testing.T) { + config := Config{ + Stream: "attachment-jobs", + Group: "attachment-workers", + Consumer: "worker-1", + FailedStream: "attachment-jobs-failed", + FailedMaxLen: 1000, + Block: time.Second, + ClaimMinIdle: time.Minute, + ReadCount: 10, + } + require.NoError(t, config.validate()) + + config.ClaimMinIdle = 0 + require.Error(t, config.validate()) +} diff --git a/internal/broker/redisstream/envelope.go b/internal/broker/redisstream/envelope.go new file mode 100644 index 0000000..44ffb4a --- /dev/null +++ b/internal/broker/redisstream/envelope.go @@ -0,0 +1,147 @@ +package redisstream + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +const ( + SchemaVersion = 1 + JobType = "attachment.process.v1" +) + +var ( + errInvalidEnvelope = errors.New("invalid job envelope") + traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +) + +type Envelope struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func (e Envelope) Validate() error { + switch { + case e.SchemaVersion != SchemaVersion: + return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope) + case e.Type != JobType: + return fmt.Errorf("%w: unsupported type", errInvalidEnvelope) + case e.JobID == uuid.Nil: + return fmt.Errorf("%w: job_id is required", errInvalidEnvelope) + case e.AttachmentID == uuid.Nil: + return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope) + case e.PipelineVersion < 1: + return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope) + case e.Attempt < 1: + return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope) + case e.CreatedAt.IsZero(): + return fmt.Errorf("%w: created_at is required", errInvalidEnvelope) + case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent): + return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope) + case utf8.RuneCountInString(e.Tracestate) > 512: + return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope) + default: + return nil + } +} + +func EncodeEnvelope(envelope Envelope) ([]byte, error) { + if err := envelope.Validate(); err != nil { + return nil, err + } + + payload, err := json.Marshal(envelope) + if err != nil { + return nil, fmt.Errorf("encode job envelope: %w", err) + } + + return payload, nil +} + +func DecodeEnvelope(payload []byte) (Envelope, error) { + var wire envelopeJSON + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wire); err != nil { + return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err) + } + if err := ensureJSONEnd(decoder); err != nil { + return Envelope{}, err + } + + jobID, err := parseUUID(wire.JobID, "job_id") + if err != nil { + return Envelope{}, err + } + attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id") + if err != nil { + return Envelope{}, err + } + createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt) + if err != nil { + return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope) + } + envelope := Envelope{ + SchemaVersion: wire.SchemaVersion, + JobID: jobID, + Type: wire.Type, + AttachmentID: attachmentID, + PipelineVersion: wire.PipelineVersion, + Attempt: wire.Attempt, + CreatedAt: createdAt, + Traceparent: wire.Traceparent, + Tracestate: wire.Tracestate, + } + if err := envelope.Validate(); err != nil { + return Envelope{}, err + } + + return envelope, nil +} + +type envelopeJSON struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func parseUUID(value, field string) (uuid.UUID, error) { + if !uuidPattern.MatchString(value) { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + parsed, err := uuid.Parse(value) + if err != nil { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + return parsed, nil +} + +func ensureJSONEnd(decoder *json.Decoder) error { + var value any + if err := decoder.Decode(&value); !errors.Is(err, io.EOF) { + return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope) + } + return nil +} diff --git a/internal/broker/redisstream/envelope_test.go b/internal/broker/redisstream/envelope_test.go new file mode 100644 index 0000000..f24b68c --- /dev/null +++ b/internal/broker/redisstream/envelope_test.go @@ -0,0 +1,74 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestEnvelopeRoundTrip(t *testing.T) { + envelope := validEnvelope() + payload, err := EncodeEnvelope(envelope) + require.NoError(t, err) + + decoded, err := DecodeEnvelope(payload) + require.NoError(t, err) + require.Equal(t, envelope, decoded) +} + +func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) { + payload := []byte(`{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"must-not-enter-the-broker" + }`) + + _, err := DecodeEnvelope(payload) + require.Error(t, err) +} + +func TestEnvelopeValidation(t *testing.T) { + tests := []struct { + name string + mutate func(*Envelope) + }{ + {name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }}, + {name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }}, + {name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }}, + {name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }}, + {name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }}, + {name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }}, + {name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }}, + {name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }}, + {name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + envelope := validEnvelope() + tt.mutate(&envelope) + require.Error(t, envelope.Validate()) + }) + } +} + +func validEnvelope() Envelope { + return Envelope{ + SchemaVersion: SchemaVersion, + JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"), + Type: JobType, + AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"), + PipelineVersion: 1, + Attempt: 1, + CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC), + Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + Tracestate: "vendor=value", + } +} diff --git a/internal/config/config.go b/internal/config/config.go index dcf5b9a..977abc5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -46,6 +46,7 @@ type ServerConfig struct { AttachmentMaxPixels int64 AttachmentBodyMemory int64 AttachmentDecodeMemory int64 + PipelineVersion int } type MigrationConfig struct { @@ -97,14 +98,16 @@ type objectStorageConfig struct { } func LoadServer() (ServerConfig, error) { - values := viper.New() - values.AutomaticEnv() - - host, err := requiredString(values, "APP_HOST") + values, err := loadValues() if err != nil { return ServerConfig{}, err } - port, err := requiredPort(values, "APP_PORT") + + host, err := requiredString(values, "server.host") + if err != nil { + return ServerConfig{}, err + } + port, err := requiredPort(values, "server.port") if err != nil { return ServerConfig{}, err } @@ -112,71 +115,71 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } - cacheURL, err := requiredString(values, "CACHE_URL") + cacheURL, err := requiredString(values, "cache.url") if err != nil { return ServerConfig{}, err } - store, err := requiredString(values, "SESSION_STORE") + store, err := requiredString(values, "server.session_store") if err != nil { return ServerConfig{}, err } if store != "valkey" { - return ServerConfig{}, errors.New("SESSION_STORE must be valkey") + return ServerConfig{}, errors.New("server.session_store must be valkey") } - sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30) + sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour) if err != nil { return ServerConfig{}, err } - idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30) + idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour) if err != nil { return ServerConfig{}, err } - requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300) + requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300) + readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300) + readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300) + writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600) + idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour) if err != nil { return ServerConfig{}, err } - shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300) + shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024) + passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024) if err != nil { return ServerConfig{}, err } - passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20) + passwordIterations, err := integer(values, "server.password.iterations", 1, 20) if err != nil { return ServerConfig{}, err } - passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255) + passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255) if err != nil { return ServerConfig{}, err } - passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024) + passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024) if err != nil { return ServerConfig{}, err } - passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024) + passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024) if err != nil { return ServerConfig{}, err } - swaggerPort, err := requiredPort(values, "SWAGGER_PORT") + swaggerPort, err := requiredPort(values, "server.swagger_port") if err != nil { return ServerConfig{}, err } @@ -184,28 +187,32 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } - attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) if err != nil { return ServerConfig{}, err } - attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) if err != nil { return ServerConfig{}, err } - attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024) + attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024) if err != nil { return ServerConfig{}, err } if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { - return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes) + return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes) } - attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024) + attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024) if err != nil { return ServerConfig{}, err } minimumDecodeMemory := int64(attachmentMaxPixels) * 8 if int64(attachmentDecodeMemory) < minimumDecodeMemory { - return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) + return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory) + } + pipelineVersion, err := integer(values, "pipeline.version", 1, 1000) + if err != nil { + return ServerConfig{}, err } return ServerConfig{ @@ -226,17 +233,20 @@ func LoadServer() (ServerConfig, error) { AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), AttachmentBodyMemory: int64(attachmentBodyMemory), AttachmentDecodeMemory: int64(attachmentDecodeMemory), + PipelineVersion: pipelineVersion, }, nil } func LoadMigration() (MigrationConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return MigrationConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return MigrationConfig{}, err } - migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600) + migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute) if err != nil { return MigrationConfig{}, err } @@ -249,8 +259,10 @@ func LoadMigration() (MigrationConfig, error) { } func LoadReconciliation() (ReconciliationConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return ReconciliationConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return ReconciliationConfig{}, err @@ -270,31 +282,31 @@ func LoadReconciliation() (ReconciliationConfig, error) { } func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { - endpoint, err := requiredString(values, "S3_ENDPOINT") + endpoint, err := requiredString(values, "object_storage.endpoint") if err != nil { return objectStorageConfig{}, err } - region, err := requiredString(values, "S3_REGION") + region, err := requiredString(values, "object_storage.region") if err != nil { return objectStorageConfig{}, err } - bucket, err := requiredString(values, "S3_BUCKET") + bucket, err := requiredString(values, "object_storage.bucket") if err != nil { return objectStorageConfig{}, err } - accessKey, err := requiredString(values, "S3_ACCESS_KEY") + accessKey, err := requiredString(values, "object_storage.access_key") if err != nil { return objectStorageConfig{}, err } - secretKey, err := requiredString(values, "S3_SECRET_KEY") + secretKey, err := requiredString(values, "object_storage.secret_key") if err != nil { return objectStorageConfig{}, err } - useTLS, err := boolean(values, "S3_USE_TLS") + useTLS, err := boolean(values, "object_storage.use_tls") if err != nil { return objectStorageConfig{}, err } - timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) + timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute) if err != nil { return objectStorageConfig{}, err } @@ -305,31 +317,31 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { } func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { - databaseURL, err := requiredString(values, "DATABASE_URL") + databaseURL, err := requiredString(values, "database.url") if err != nil { return databaseConfig{}, err } - maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100) + maxConns, err := integer(values, "database.max_connections", 1, 100) if err != nil { return databaseConfig{}, err } - minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns) + minConns, err := integer(values, "database.min_connections", 0, maxConns) if err != nil { return databaseConfig{}, err } - maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400) + maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour) if err != nil { return databaseConfig{}, err } - maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400) + maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour) if err != nil { return databaseConfig{}, err } - healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600) + healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour) if err != nil { return databaseConfig{}, err } - dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60) + dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute) if err != nil { return databaseConfig{}, err } @@ -359,12 +371,16 @@ func requiredPort(values *viper.Viper, name string) (string, error) { return value, nil } -func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) { - value, err := integer(values, name, minimum, maximum) +func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) { + value, err := requiredString(values, name) if err != nil { return 0, err } - return time.Duration(value) * time.Second, nil + parsed, err := time.ParseDuration(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum) + } + return parsed, nil } func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 863746c..80d2216 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,24 +1,17 @@ package config import ( + "os" + "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { - values := map[string]string{ - "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", - "DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4", - "DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60", - "DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5", - "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", - "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", - "S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5", - } - for name, value := range values { - t.Setenv(name, value) - } + setConfigEnvironment(t, validTestConfig) t.Setenv("APP_HOST", "") t.Setenv("CACHE_URL", "") t.Setenv("SWAGGER_PORT", "") @@ -27,3 +20,112 @@ func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { require.NoError(t, err) require.Equal(t, "attachments", settings.S3Bucket) } + +func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("WORKER_CONCURRENCY", "64") + + settings, err := LoadWorker() + require.NoError(t, err) + require.Equal(t, 2, settings.Concurrency) + require.Equal(t, 5*time.Second, settings.HeartbeatInterval) +} + +func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) { + content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1) + setConfigEnvironment(t, content) + + _, err := LoadWorker() + require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease") +} + +func TestLoadValuesRejectsUnknownVersion(t *testing.T) { + setConfigEnvironment(t, "version: 2\n") + + _, err := loadValues() + require.ErrorContains(t, err, "version: must be 1") +} + +func setConfigEnvironment(t *testing.T, content string) { + t.Helper() + path := filepath.Join(t.TempDir(), "messenger.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + t.Setenv("MESSENGER_CONFIG_PATH", path) + for name, value := range map[string]string{ + "APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081", + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + } { + t.Setenv(name, value) + } +} + +const validTestConfig = `version: 1 +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 +database: + min_connections: 0 + max_connections: 4 + max_connection_age: 5m + max_connection_idle: 1m + health_check_period: 30s + dependency_timeout: 5s +migration: + timeout: 30s +object_storage: + use_tls: false + timeout: 5s +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 +pipeline: + version: 1 +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 +monitoring: + metrics_address: 0.0.0.0:9091 +` diff --git a/internal/config/loader.go b/internal/config/loader.go new file mode 100644 index 0000000..6cd115c --- /dev/null +++ b/internal/config/loader.go @@ -0,0 +1,47 @@ +package config + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/viper" +) + +const supportedConfigVersion = 1 + +var environmentKeys = map[string]string{ + "server.host": "APP_HOST", + "server.port": "APP_PORT", + "server.swagger_port": "SWAGGER_PORT", + "database.url": "DATABASE_URL", + "cache.url": "CACHE_URL", + "object_storage.endpoint": "S3_ENDPOINT", + "object_storage.region": "S3_REGION", + "object_storage.bucket": "S3_BUCKET", + "object_storage.access_key": "S3_ACCESS_KEY", + "object_storage.secret_key": "S3_SECRET_KEY", + "broker.url": "BROKER_URL", +} + +func loadValues() (*viper.Viper, error) { + path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH")) + if path == "" { + return nil, errors.New("MESSENGER_CONFIG_PATH is required") + } + values := viper.New() + values.SetConfigFile(path) + if err := values.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read messenger config: %w", err) + } + if version := values.GetInt("version"); version != supportedConfigVersion { + return nil, fmt.Errorf("version: must be %d", supportedConfigVersion) + } + for key, name := range environmentKeys { + if err := values.BindEnv(key, name); err != nil { + return nil, fmt.Errorf("bind %s: %w", name, err) + } + } + return values, nil +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go new file mode 100644 index 0000000..76b4f5d --- /dev/null +++ b/internal/config/pipeline.go @@ -0,0 +1,307 @@ +package config + +import ( + "errors" + "fmt" + "net" + "time" + + "github.com/spf13/viper" +) + +type DispatcherConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int + MetricsAddress string +} + +type WorkerConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + Concurrency int + Prefetch int + LeaseDuration time.Duration + HeartbeatInterval time.Duration + ProcessingTimeout time.Duration + ReadBlock time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + MemoryBytes int64 + MetricsAddress string +} + +type JobAdminConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration +} + +type brokerConfig struct { + url string + stream string + consumerGroup string + failedStream string + failedStreamMaxLength int +} + +func LoadJobAdmin() (JobAdminConfig, error) { + values, err := loadValues() + if err != nil { + return JobAdminConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return JobAdminConfig{}, err + } + return JobAdminConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + }, nil +} + +func LoadDispatcher() (DispatcherConfig, error) { + values, err := loadValues() + if err != nil { + return DispatcherConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + batchSize, err := integer(values, "outbox.batch_size", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute) + if err != nil { + return DispatcherConfig{}, err + } + pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute) + if err != nil { + return DispatcherConfig{}, err + } + recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return DispatcherConfig{}, err + } + return DispatcherConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, + FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, + RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, + Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize, + MetricsAddress: metricsAddress, + }, nil +} + +func LoadWorker() (WorkerConfig, error) { + values, err := loadValues() + if err != nil { + return WorkerConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return WorkerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return WorkerConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return WorkerConfig{}, err + } + concurrency, err := integer(values, "worker.concurrency", 1, 64) + if err != nil { + return WorkerConfig{}, err + } + prefetch, err := integer(values, "worker.prefetch", concurrency, 1024) + if err != nil { + return WorkerConfig{}, err + } + leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if heartbeatInterval >= leaseDuration { + return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease") + } + processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if processingTimeout >= leaseDuration { + return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease") + } + readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute) + if err != nil { + return WorkerConfig{}, err + } + reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + maxAttempts, err := integer(values, "worker.max_attempts", 1, 100) + if err != nil { + return WorkerConfig{}, err + } + retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if retryMax < retryBase { + return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base") + } + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) + if err != nil { + return WorkerConfig{}, err + } + memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return WorkerConfig{}, err + } + return WorkerConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS, + S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream, + ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, + FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, + HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock, + ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress, + }, nil +} + +func metricsAddress(values *viper.Viper) (string, error) { + address, err := requiredString(values, "monitoring.metrics_address") + if err != nil { + return "", err + } + _, port, err := net.SplitHostPort(address) + if err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + return address, nil +} + +func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { + url, err := requiredString(values, "broker.url") + if err != nil { + return brokerConfig{}, err + } + stream, err := requiredString(values, "broker.stream") + if err != nil { + return brokerConfig{}, err + } + consumerGroup, err := requiredString(values, "broker.consumer_group") + if err != nil { + return brokerConfig{}, err + } + failedStream, err := requiredString(values, "broker.failed_stream") + if err != nil { + return brokerConfig{}, err + } + failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000) + if err != nil { + return brokerConfig{}, err + } + return brokerConfig{ + url: url, stream: stream, consumerGroup: consumerGroup, + failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength, + }, nil +} diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go new file mode 100644 index 0000000..8e8d432 --- /dev/null +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -0,0 +1,104 @@ +package pipeline + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} + +type Metrics struct { + workerInFlight prometheus.Gauge + workerJobs *prometheus.CounterVec + workerDuration *prometheus.HistogramVec + workerRetries prometheus.Counter + workerReclaimed prometheus.Counter + outboxPublished prometheus.Counter + outboxPublishFailure prometheus.Counter + outboxRecovered prometheus.Counter + handler http.Handler +} + +func NewMetrics() *Metrics { + registry := prometheus.NewRegistry() + metrics := &Metrics{ + workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight", + Help: "Attachment jobs currently processed by this worker.", + }), + workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total", + Help: "Completed attachment job deliveries by result.", + }, []string{"result"}), + workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds", + Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets, + }, []string{"result"}), + workerRetries: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total", + Help: "Attachment jobs scheduled for another attempt.", + }), + workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total", + Help: "Redis Stream deliveries reclaimed after their idle timeout.", + }), + outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "outbox", Name: "published_total", + Help: "Outbox events confirmed by Redis Streams.", + }), + outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total", + Help: "Outbox events whose Redis Streams publish failed.", + }), + outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total", + Help: "Expired outbox leases returned for publication.", + }), + } + registry.MustRegister( + metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration, + metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished, + metrics.outboxPublishFailure, metrics.outboxRecovered, + collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + return metrics +} + +func (m *Metrics) Handler() http.Handler { + return m.handler +} + +func (m *Metrics) ObserveStarted() { + m.workerInFlight.Inc() +} + +func (m *Metrics) ObserveFinished(result string, duration time.Duration) { + m.workerInFlight.Dec() + m.workerJobs.WithLabelValues(result).Inc() + m.workerDuration.WithLabelValues(result).Observe(duration.Seconds()) +} + +func (m *Metrics) ObserveRetry() { + m.workerRetries.Inc() +} + +func (m *Metrics) ObserveReclaimed(count int) { + m.workerReclaimed.Add(float64(count)) +} + +func (m *Metrics) ObservePublished() { + m.outboxPublished.Inc() +} + +func (m *Metrics) ObservePublishFailure() { + m.outboxPublishFailure.Inc() +} + +func (m *Metrics) ObserveRecovered(count int) { + m.outboxRecovered.Add(float64(count)) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go index f5a1c96..7e0bbf8 100644 --- a/internal/messenger/attachment/service.go +++ b/internal/messenger/attachment/service.go @@ -25,10 +25,10 @@ import ( type Repository interface { ChatForUser(context.Context, string, string) (model.Chat, error) CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error) - MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) + QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error) DeletePendingAttachment(context.Context, string, string) error - AttachmentByStorage(context.Context, string, string) (model.Attachment, error) AttachmentForUser(context.Context, string, string) (model.Attachment, error) + AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) } type ObjectStore interface { @@ -43,6 +43,7 @@ type Config struct { BodyMemoryBytes int64 DecodeMemoryBytes int64 CleanupTimeout time.Duration + PipelineVersion int } type Upload struct { @@ -52,10 +53,13 @@ type Upload struct { ContentType string ContentLength int64 Body io.Reader + Traceparent string + Tracestate string } type Content struct { Attachment model.Attachment + Variant model.Variant Body io.ReadCloser } @@ -69,7 +73,7 @@ type Service struct { } func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { - if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 { return nil, errors.New("invalid attachment service configuration") } if config.BodyMemoryBytes < config.MaxBytes { @@ -96,58 +100,38 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e defer release() attachmentID := uuid.NewString() - objectKey := "attachments/" + uuid.NewString() + "/original" + stagingKey := "staging/" + attachmentID + "/" + uuid.NewString() digest := sha256.Sum256(data) pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{ ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename, ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), - ObjectKey: objectKey, Status: model.AttachmentPending, + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, + PipelineVersion: s.config.PipelineVersion, }) if err != nil { return model.Attachment{}, mapDependencyError(err) } - if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil { - s.deletePending(pending.ID, objectKey) + if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil { + s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err) + s.deletePending(pending.ID, stagingKey) return model.Attachment{}, model.ErrUnavailable } - attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey) - if err == nil { - return attachment, nil + createdAt := time.Now().UTC() + queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion, + Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt, + }, input.Traceparent, input.Tracestate) + if err != nil { + s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err) + return model.Attachment{}, mapDependencyError(err) } - return s.resolveReadyOutcome(pending.ID, objectKey) + return queued, nil } -func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) { +func (s *Service) deletePending(attachmentID, stagingKey string) { cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) defer cancel() - stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey) - if err == nil && stored.Status == model.AttachmentReady { - return stored, nil - } - if err == nil && stored.Status == model.AttachmentPending { - if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { - s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr) - return model.Attachment{}, model.ErrUnavailable - } - if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil { - s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr) - } - return model.Attachment{}, model.ErrUnavailable - } - if errors.Is(err, model.ErrNotFound) { - if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil { - s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr) - } - return model.Attachment{}, model.ErrUnavailable - } - s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err) - return model.Attachment{}, model.ErrUnavailable -} - -func (s *Service) deletePending(attachmentID, objectKey string) { - cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) - defer cancel() - if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil { + if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil { s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) } } @@ -160,16 +144,16 @@ func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (mo return attachment, nil } -func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) { - attachment, err := s.Metadata(ctx, attachmentID, userID) +func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) { + attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName) if err != nil { - return Content{}, err + return Content{}, mapDependencyError(err) } - body, err := s.objects.Get(ctx, attachment.ObjectKey) + body, err := s.objects.Get(ctx, variant.ObjectKey) if err != nil { return Content{}, model.ErrUnavailable } - return Content{Attachment: attachment, Body: body}, nil + return Content{Attachment: attachment, Variant: variant, Body: body}, nil } func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go index f06a895..da14eb0 100644 --- a/internal/messenger/attachment/service_test.go +++ b/internal/messenger/attachment/service_test.go @@ -27,8 +27,8 @@ func TestUploadLifecycle(t *testing.T) { result, err := service.Upload(context.Background(), upload(data, "image/png")) require.NoError(t, err) - require.Equal(t, model.AttachmentReady, result.Status) - require.Equal(t, []string{"pending", "put", "ready"}, events) + require.Equal(t, model.AttachmentQueued, result.Status) + require.Equal(t, []string{"pending", "put", "queued"}, events) } func TestUploadValidation(t *testing.T) { @@ -86,7 +86,7 @@ func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) { slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{}, Config{ MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024, - DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, + DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1, }, ) require.Error(t, err) @@ -103,16 +103,16 @@ func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { require.True(t, repository.pendingDeleted) } -func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) { +func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) { data := pngFixture(t, 2, 2) - repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true} + repository := &repositoryStub{queueErr: errors.New("connection lost")} objects := &objectStub{} service := newTestService(t, repository, objects, 1024, 4) - result, err := service.Upload(context.Background(), upload(data, "image/png")) - require.NoError(t, err) - require.Equal(t, model.AttachmentReady, result.Status) + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) require.False(t, objects.deleted) + require.False(t, repository.pendingDeleted) } func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service { @@ -121,7 +121,7 @@ func newTestService(t *testing.T, repository Repository, objects ObjectStore, ma slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects, Config{ MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes, - DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, + DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1, }, ) require.NoError(t, err) @@ -137,8 +137,7 @@ func upload(data []byte, contentType string) Upload { type repositoryStub struct { events *[]string - markErr error - lookupReady bool + queueErr error pendingDeleted bool pending model.Attachment } @@ -154,12 +153,19 @@ func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment m return attachment, nil } -func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) { - if s.markErr != nil { - return model.Attachment{}, s.markErr +func (s *repositoryStub) QueueAttachmentWithJobAndOutbox( + context.Context, + string, + string, + model.AttachmentJob, + string, + string, +) (model.Attachment, error) { + if s.queueErr != nil { + return model.Attachment{}, s.queueErr } - s.pending.Status = model.AttachmentReady - s.record("ready") + s.pending.Status = model.AttachmentQueued + s.record("queued") return s.pending, nil } @@ -168,18 +174,14 @@ func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string return nil } -func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) { - if s.lookupReady { - s.pending.Status = model.AttachmentReady - return s.pending, nil - } - return model.Attachment{}, model.ErrNotFound -} - func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { return s.pending, nil } +func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) { + return s.pending, model.Variant{}, nil +} + func (s *repositoryStub) record(event string) { if s.events != nil { *s.events = append(*s.events, event) diff --git a/internal/messenger/attachment/transform/exif.go b/internal/messenger/attachment/transform/exif.go new file mode 100644 index 0000000..53e532b --- /dev/null +++ b/internal/messenger/attachment/transform/exif.go @@ -0,0 +1,169 @@ +package transform + +import ( + "context" + "encoding/binary" + "errors" + "image" + "image/color" +) + +var errInvalidEXIF = errors.New("invalid EXIF metadata") + +func jpegOrientation(ctx context.Context, data []byte) (int, error) { + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + return 0, errInvalidEXIF + } + for offset := 2; offset < len(data); { + if err := contextError(ctx); err != nil { + return 0, err + } + if data[offset] != 0xff { + return 0, errInvalidEXIF + } + for offset < len(data) && data[offset] == 0xff { + offset++ + } + if offset >= len(data) { + return 0, errInvalidEXIF + } + marker := data[offset] + offset++ + if marker == 0xd9 || marker == 0xda { + return 1, nil + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + if offset+2 > len(data) { + return 0, errInvalidEXIF + } + length := int(binary.BigEndian.Uint16(data[offset : offset+2])) + if length < 2 || offset+length > len(data) { + return 0, errInvalidEXIF + } + payload := data[offset+2 : offset+length] + offset += length + if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" { + continue + } + return tiffOrientation(ctx, payload[6:]) + } + return 1, nil +} + +func tiffOrientation(ctx context.Context, data []byte) (int, error) { + if len(data) < 8 { + return 0, errInvalidEXIF + } + var order binary.ByteOrder + switch string(data[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 0, errInvalidEXIF + } + if order.Uint16(data[2:4]) != 42 { + return 0, errInvalidEXIF + } + ifdOffset := uint64(order.Uint32(data[4:8])) + if ifdOffset > uint64(len(data)-2) { + return 0, errInvalidEXIF + } + count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2])) + entriesStart := ifdOffset + 2 + if count > (uint64(len(data))-entriesStart)/12 { + return 0, errInvalidEXIF + } + for index := uint64(0); index < count; index++ { + if err := contextError(ctx); err != nil { + return 0, err + } + entry := data[entriesStart+index*12 : entriesStart+(index+1)*12] + if order.Uint16(entry[:2]) != 0x0112 { + continue + } + if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 { + return 0, errInvalidEXIF + } + orientation := int(order.Uint16(entry[8:10])) + if orientation < 1 || orientation > 8 { + return 0, errInvalidEXIF + } + return orientation, nil + } + return 1, nil +} + +func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) { + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + outputWidth, outputHeight := width, height + if orientation >= 5 { + outputWidth, outputHeight = height, width + } + output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight)) + for y := 0; y < outputHeight; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < outputWidth; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation) + output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY)) + } + } + return output, nil +} + +func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { + switch orientation { + case 2: + return width - 1 - x, y + case 3: + return width - 1 - x, height - 1 - y + case 4: + return x, height - 1 - y + case 5: + return y, x + case 6: + return y, height - 1 - x + case 7: + return width - 1 - y, height - 1 - x + case 8: + return width - 1 - y, x + default: + return x, y + } +} + +func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() + output.SetNRGBA(x, y, color.NRGBA{ + R: uint8((r + 0xffff - a) / 0x101), + G: uint8((g + 0xffff - a) / 0x101), + B: uint8((b + 0xffff - a) / 0x101), + A: 0xff, + }) + } + } + return output, nil +} diff --git a/internal/messenger/attachment/transform/processor.go b/internal/messenger/attachment/transform/processor.go new file mode 100644 index 0000000..8a6dd08 --- /dev/null +++ b/internal/messenger/attachment/transform/processor.go @@ -0,0 +1,272 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/jpeg" + _ "image/png" + "io" + "math" + "net/http" + + "github.com/deepteams/webp" +) + +const ( + // DefaultJPEGQuality is used for thumbnail and crop variants. + DefaultJPEGQuality = 85 + // DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size. + DefaultWebPQuality = 82 + // DefaultWebPMethod selects the encoder's balanced quality and CPU mode. + DefaultWebPMethod = 4 +) + +var ( + ErrInvalidConfig = errors.New("invalid image transform configuration") + ErrUnsupportedMedia = errors.New("unsupported image media type") + ErrPayloadTooLarge = errors.New("image exceeds configured limits") + ErrInvalidImage = errors.New("invalid image data") +) + +type Config struct { + MaxBytes int64 + MaxPixels int64 + JPEGQuality int + WebPQuality float32 + WebPMethod int +} + +type Variant struct { + Bytes []byte + ContentType string + Width int + Height int + SHA256 string +} + +type Result struct { + Original Variant + Thumbnail Variant + Crop Variant + Optimized Variant + Orientation int +} + +type Processor struct { + config Config +} + +func DefaultConfig(maxBytes, maxPixels int64) Config { + return Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, + JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod, + } +} + +func NewProcessor(config Config) (*Processor, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 || + config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 { + return nil, ErrInvalidConfig + } + return &Processor{config: config}, nil +} + +func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) { + if err := contextError(ctx); err != nil { + return Result{}, err + } + img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + oriented, err := applyOrientation(ctx, img, orientation) + if err != nil { + return Result{}, err + } + opaque, err := flattenWhite(ctx, oriented) + if err != nil { + return Result{}, err + } + result := Result{ + Original: variant(data, format, width, height), + Orientation: orientation, + } + + thumbnail, err := fitContext(ctx, opaque, 256, 256, false) + if err != nil { + return Result{}, err + } + result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + crop := centeredSquare(opaque) + crop, err = resizeContext(ctx, crop, 128, 128) + if err != nil { + return Result{}, err + } + result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + optimized, err := fitContext(ctx, opaque, 1280, 1280, false) + if err != nil { + return Result{}, err + } + result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + return result, nil +} + +func (p *Processor) decode( + ctx context.Context, + data []byte, + declaredContentType string, +) (image.Image, string, int, int, int, error) { + if declaredContentType != "image/jpeg" && declaredContentType != "image/png" { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + if len(data) == 0 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(len(data)) > p.config.MaxBytes { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + if http.DetectContentType(data) != declaredContentType { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(config.Width) > p.config.MaxPixels/int64(config.Height) { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + if err != nil || decodedFormat != format { + return nil, "", 0, 0, 0, ErrInvalidImage + } + orientation := 1 + if format == "jpeg" { + orientation, err = jpegOrientation(ctx, data) + if err != nil { + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + return nil, "", 0, 0, 0, ErrInvalidImage + } + } + return img, declaredContentType, config.Width, config.Height, orientation, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + if err := contextError(r.ctx); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + +func contextError(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +func contentType(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + default: + return "" + } +} + +func encodeJPEG(img image.Image, quality int) (Variant, error) { + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil +} + +func encodeWebP(img image.Image, quality float32, method int) (Variant, error) { + var output bytes.Buffer + options := &webp.EncoderOptions{Quality: quality, Method: method} + if err := webp.Encode(&output, img, options); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil +} + +func variant(data []byte, contentType string, width, height int) Variant { + digest := sha256.Sum256(data) + return Variant{ + Bytes: data, ContentType: contentType, Width: width, Height: height, + SHA256: hex.EncodeToString(digest[:]), + } +} + +// EstimateMemoryWeight returns a conservative reservation for the input, decoded source, +// orientation and alpha copies, variant working buffer, and encoded variant buffers. +func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) { + if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) { + return 0, ErrInvalidImage + } + pixels := int64(width) * int64(height) + optimizedPixels := fittedPixels(width, height, 1280, 1280) + thumbnailPixels := fittedPixels(width, height, 256, 256) + const cropPixels int64 = 128 * 128 + if pixels > (math.MaxInt64-inputBytes)/16 { + return 0, ErrPayloadTooLarge + } + weight := inputBytes + pixels*16 + variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4 + if variants > math.MaxInt64-weight { + return 0, ErrPayloadTooLarge + } + return weight + variants, nil +} + +func fittedPixels(width, height, maxWidth, maxHeight int) int64 { + newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false) + return int64(newWidth) * int64(newHeight) +} diff --git a/internal/messenger/attachment/transform/processor_test.go b/internal/messenger/attachment/transform/processor_test.go new file mode 100644 index 0000000..7b595dd --- /dev/null +++ b/internal/messenger/attachment/transform/processor_test.go @@ -0,0 +1,261 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "testing" + + "github.com/deepteams/webp" +) + +func TestProcessProducesExpectedVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 320*200) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Original, "image/png", 320, 200) + assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) + assertVariant(t, result.Optimized, "image/webp", 320, 200) + if !bytes.Equal(result.Original.Bytes, data) { + t.Fatal("original bytes changed") + } + digest := sha256.Sum256(data) + if result.Original.SHA256 != hex.EncodeToString(digest[:]) { + t.Fatalf("unexpected original digest: %s", result.Original.SHA256) + } + if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil { + t.Fatalf("decode optimized WebP: %v", err) + } +} + +func TestProcessDoesNotUpscaleFitVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 64, 32)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 64*32) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32) + assertVariant(t, result.Optimized, "image/webp", 64, 32) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) +} + +func TestFlattenWhiteCompositesAlpha(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0}) + source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128}) + + result, err := flattenWhite(context.Background(), source) + if err != nil { + t.Fatalf("flatten image: %v", err) + } + if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) { + t.Fatalf("transparent pixel: got %#v", got) + } + if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) { + t.Fatalf("semi-transparent pixel: got %#v", got) + } +} + +func TestProcessAppliesEXIFOrientation(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + fillGradient(source) + data := injectOrientation(t, encodeJPEGForTest(t, source), 6) + processor := newTestProcessor(t, int64(len(data))+1, 3*2) + + result, err := processor.Process(context.Background(), data, "image/jpeg") + if err != nil { + t.Fatalf("process image: %v", err) + } + + if result.Orientation != 6 { + t.Fatalf("orientation: got %d, want 6", result.Orientation) + } + assertVariant(t, result.Original, "image/jpeg", 3, 2) + assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3) + assertVariant(t, result.Optimized, "image/webp", 2, 3) +} + +func TestProcessRejectsInvalidInputs(t *testing.T) { + valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10))) + + tests := []struct { + name string + data []byte + contentType string + maxPixels int64 + want error + }{ + {name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge}, + {name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels) + _, err := processor.Process(context.Background(), test.data, test.contentType) + if !errors.Is(err, test.want) { + t.Fatalf("error: got %v, want %v", err, test.want) + } + }) + } +} + +func TestProcessIsDeterministic(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 48, 31)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 48*31) + + first, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("first process: %v", err) + } + second, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("second process: %v", err) + } + + for name, variants := range map[string][2]Variant{ + "thumbnail": {first.Thumbnail, second.Thumbnail}, + "crop": {first.Crop, second.Crop}, + "optimized": {first.Optimized, second.Optimized}, + } { + if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) { + t.Fatalf("%s bytes differ", name) + } + } +} + +func TestProcessHonorsCanceledContext(t *testing.T) { + processor := newTestProcessor(t, 1024, 100) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := processor.Process(ctx, []byte("unused"), "image/png") + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +func TestResizeStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := cancelingImage{ + Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)), + cancel: cancel, + } + _, err := resizeContext(ctx, source, 128, 128) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +type cancelingImage struct { + image.Image + cancel context.CancelFunc +} + +func (i cancelingImage) At(x, y int) color.Color { + i.cancel() + return i.Image.At(x, y) +} + +func TestEstimateMemoryWeight(t *testing.T) { + weight, err := EstimateMemoryWeight(1000, 320, 200) + if err != nil { + t.Fatalf("estimate memory: %v", err) + } + want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4) + if weight != want { + t.Fatalf("weight: got %d, want %d", weight, want) + } +} + +func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor { + t.Helper() + processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels)) + if err != nil { + t.Fatalf("new processor: %v", err) + } + return processor +} + +func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) { + t.Helper() + if variant.ContentType != contentType || variant.Width != width || variant.Height != height { + t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d", + variant.ContentType, variant.Width, variant.Height, contentType, width, height) + } + if len(variant.Bytes) == 0 || variant.SHA256 == "" { + t.Fatal("variant has no bytes or digest") + } +} + +func fillGradient(img *image.NRGBA) { + for y := 0; y < img.Bounds().Dy(); y++ { + for x := 0; x < img.Bounds().Dx(); x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } +} + +func encodePNG(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := png.Encode(&output, img); err != nil { + t.Fatalf("encode PNG: %v", err) + } + return output.Bytes() +} + +func encodeJPEGForTest(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil { + t.Fatalf("encode JPEG: %v", err) + } + return output.Bytes() +} + +func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte { + t.Helper() + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + t.Fatal("not a JPEG") + } + tiff := make([]byte, 26) + copy(tiff[:2], "II") + binary.LittleEndian.PutUint16(tiff[2:4], 42) + binary.LittleEndian.PutUint32(tiff[4:8], 8) + binary.LittleEndian.PutUint16(tiff[8:10], 1) + binary.LittleEndian.PutUint16(tiff[10:12], 0x0112) + binary.LittleEndian.PutUint16(tiff[12:14], 3) + binary.LittleEndian.PutUint32(tiff[14:18], 1) + binary.LittleEndian.PutUint16(tiff[18:20], orientation) + payload := append([]byte("Exif\x00\x00"), tiff...) + segment := []byte{0xff, 0xe1, 0, 0} + binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2)) + result := make([]byte, 0, len(data)+len(segment)+len(payload)) + result = append(result, data[:2]...) + result = append(result, segment...) + result = append(result, payload...) + return append(result, data[2:]...) +} diff --git a/internal/messenger/attachment/transform/resize.go b/internal/messenger/attachment/transform/resize.go new file mode 100644 index 0000000..2d86bb8 --- /dev/null +++ b/internal/messenger/attachment/transform/resize.go @@ -0,0 +1,128 @@ +package transform + +import ( + "context" + "image" + "image/color" +) + +func fitContext( + ctx context.Context, + source image.Image, + maxWidth int, + maxHeight int, + upscale bool, +) (*image.NRGBA, error) { + bounds := source.Bounds() + width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale) + return resizeContext(ctx, source, width, height) +} + +func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) { + if !upscale && width <= maxWidth && height <= maxHeight { + return width, height + } + if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) { + newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width))) + return maxWidth, newHeight + } + newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height))) + return newWidth, maxHeight +} + +func centeredSquare(source image.Image) image.Image { + bounds := source.Bounds() + size := min(bounds.Dx(), bounds.Dy()) + left := bounds.Min.X + (bounds.Dx()-size)/2 + top := bounds.Min.Y + (bounds.Dy()-size)/2 + return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)} +} + +type croppedImage struct { + source image.Image + bounds image.Rectangle + offset image.Point +} + +func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() } +func (i croppedImage) Bounds() image.Rectangle { return i.bounds } +func (i croppedImage) At(x, y int) color.Color { + return i.source.At(i.offset.X+x, i.offset.Y+y) +} + +func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) { + bounds := source.Bounds() + if bounds.Dx() == width && bounds.Dy() == height { + return copyNRGBAContext(ctx, source) + } + output := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + y0, y1, fy := sampleCoordinate(y, height, bounds.Dy()) + for x := 0; x < width; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + x0, x1, fx := sampleCoordinate(x, width, bounds.Dx()) + c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA) + c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA) + c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA) + c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA) + output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy)) + } + } + return output, nil +} + +func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) { + coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15 + if coordinate <= 0 { + return 0, min(1, sourceSize-1), 0 + } + maximum := int64(sourceSize-1) << 16 + if coordinate >= maximum { + return sourceSize - 1, sourceSize - 1, 0 + } + first := int(coordinate >> 16) + return first, first + 1, uint32(coordinate & 0xffff) +} + +func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA { + return color.NRGBA{ + R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy), + G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy), + B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy), + A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy), + } +} + +func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 { + const one uint64 = 1 << 16 + top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx) + bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx) + value := top*(one-uint64(fy)) + bottom*uint64(fy) + return uint8((value + 1<<31) >> 32) +} + +func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y)) + } + } + return output, nil +} diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go new file mode 100644 index 0000000..f94f525 --- /dev/null +++ b/internal/messenger/attachment/worker/worker.go @@ -0,0 +1,508 @@ +package worker + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "path" + "strings" + "sync" + "time" + + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" +) + +type Repository interface { + AttachmentJob(context.Context, string) (model.AttachmentJob, error) + ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) + RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) + CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) + RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error) + FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) + RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error) +} + +type Broker interface { + Read(context.Context) ([]redisstream.Delivery, error) + Reclaim(context.Context, string) ([]redisstream.Delivery, string, error) +} + +type ObjectStore interface { + Get(context.Context, string) (io.ReadCloser, error) + Put(context.Context, string, string, []byte) error + Delete(context.Context, string) error +} + +type ImageProcessor interface { + Process(context.Context, []byte, string) (transform.Result, error) +} + +type Metrics interface { + ObserveStarted() + ObserveFinished(string, time.Duration) + ObserveRetry() + ObserveReclaimed(int) +} + +type Config struct { + WorkerID string + Concurrency int + LeaseDuration time.Duration + HeartbeatInterval time.Duration + ProcessingTimeout time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + MaxBytes int64 + MaxPixels int64 + MemoryBytes int64 +} + +type FaultPoint string + +const ( + FaultNone FaultPoint = "" + FaultBeforeSideEffect FaultPoint = "before_side_effect" + FaultAfterSideEffect FaultPoint = "after_side_effect" + FaultAfterCommit FaultPoint = "after_commit" +) + +var ErrInjectedFailure = errors.New("injected worker failure") + +type Worker struct { + logger *slog.Logger + repository Repository + broker Broker + objects ObjectStore + processor ImageProcessor + metrics Metrics + config Config + memory *semaphore.Weighted + now func() time.Time +} + +func New( + logger *slog.Logger, + repository Repository, + broker Broker, + objects ObjectStore, + processor ImageProcessor, + metrics Metrics, + config Config, +) (*Worker, error) { + if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil { + return nil, errors.New("worker dependencies are required") + } + if err := validateConfig(config); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Worker{ + logger: logger, repository: repository, broker: broker, objects: objects, + processor: processor, metrics: metrics, config: config, + memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now, + }, nil +} + +func validateConfig(config Config) error { + if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 || + config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration || + config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration || + config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 || + config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 { + return errors.New("invalid worker configuration") + } + const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128 + if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 || + config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes { + return errors.New("worker memory must fit one maximum image") + } + return nil +} + +func (w *Worker) Run(ctx context.Context) error { + lastReclaim := time.Time{} + reclaimCursor := "0-0" + for ctx.Err() == nil { + deliveries, err := w.broker.Read(ctx) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.processBatch(ctx, deliveries) + if w.now().Sub(lastReclaim) < w.config.ReclaimInterval { + continue + } + recovery, err := w.repository.RecoverExpiredAttachmentJobs( + ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase), + ) + if err != nil { + return fmt.Errorf("recover expired attachment jobs: %w", err) + } + if recovery.Requeued > 0 || recovery.Failed > 0 { + w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed) + } + deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.metrics.ObserveReclaimed(len(deliveries)) + w.processBatch(ctx, deliveries) + reclaimCursor = next + if reclaimCursor == "" { + reclaimCursor = "0-0" + } + lastReclaim = w.now() + } + return nil +} + +func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) { + limit := make(chan struct{}, w.config.Concurrency) + var group sync.WaitGroup + for _, delivery := range deliveries { + if ctx.Err() != nil { + break + } + limit <- struct{}{} + group.Add(1) + go func(delivery redisstream.Delivery) { + defer group.Done() + defer func() { <-limit }() + if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil { + w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err) + } + }(delivery) + } + group.Wait() +} + +func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error { + started := w.now() + w.metrics.ObserveStarted() + result := "transient_error" + defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }() + + if delivery.DecodeError != nil { + result = "invalid_job" + return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + } + envelope := delivery.Envelope + lease, err := w.repository.ClaimAttachmentJob( + ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt, + w.config.WorkerID, w.config.LeaseDuration, + ) + if err != nil { + result, err = w.resolveClaim(ctx, delivery, envelope, err) + return err + } + if fault == FaultBeforeSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + + processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout) + heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx) + heartbeatDone := make(chan error, 1) + go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }() + variants, err := w.process(processingCtx, lease) + cancelProcessing() + cancelHeartbeat() + heartbeatErr := <-heartbeatDone + if heartbeatErr != nil { + return heartbeatErr + } + if err != nil { + result, err = w.resolveProcessingFailure(ctx, delivery, lease, err) + return err + } + if fault == FaultAfterSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants) + if err != nil { + return fmt.Errorf("complete attachment job: %w", err) + } + if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") { + if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil { + w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) + } + } + if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale { + return fmt.Errorf("unexpected completion result %q", mutation) + } + if fault == FaultAfterCommit { + result = "injected_failure" + return ErrInjectedFailure + } + if err := delivery.Ack(ctx); err != nil { + return err + } + result = "ready" + return nil +} + +func (w *Worker) maintainLease( + ctx context.Context, + cancelProcessing context.CancelFunc, + lease model.AttachmentJobLease, +) error { + ticker := time.NewTicker(w.config.HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration) + if err != nil { + if ctx.Err() != nil { + return nil + } + cancelProcessing() + return fmt.Errorf("renew attachment job lease: %w", err) + } + if !renewed { + cancelProcessing() + return model.ErrStaleLease + } + } + } +} + +func (w *Worker) resolveClaim( + ctx context.Context, + delivery redisstream.Delivery, + envelope redisstream.Envelope, + claimErr error, +) (string, error) { + switch { + case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput): + return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobBusy): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobNotDue): + return "not_due", nil + case errors.Is(claimErr, model.ErrConflict): + job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String()) + if err != nil { + return "transient_error", fmt.Errorf("inspect conflicting job: %w", err) + } + switch job.Status { + case model.AttachmentJobReady: + return "duplicate", delivery.Ack(ctx) + case model.AttachmentJobFailed: + code := "processing_failed" + if job.ErrorCode != nil { + code = *job.ErrorCode + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } +} + +func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) { + body, err := w.objects.Get(ctx, lease.Attachment.StagingKey) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > w.config.MaxBytes { + return nil, transform.ErrPayloadTooLarge + } + imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, transform.ErrInvalidImage + } + weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height) + if err != nil { + return nil, err + } + if weight > w.config.MemoryBytes { + return nil, errWorkerCapacity + } + if err := w.memory.Acquire(ctx, weight); err != nil { + return nil, err + } + defer w.memory.Release(weight) + + processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType) + if err != nil { + return nil, err + } + variants := []struct { + name string + value transform.Variant + }{ + {name: "original", value: processed.Original}, + {name: "thumbnail", value: processed.Thumbnail}, + {name: "crop", value: processed.Crop}, + {name: "optimized", value: processed.Optimized}, + } + result := make([]model.Variant, 0, len(variants)) + for _, item := range variants { + objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value) + if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil { + return nil, err + } + result = append(result, model.Variant{ + Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)), + SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height, + ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion, + }) + } + return result, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + return r.reader.Read(buffer) + } +} + +var errWorkerCapacity = errors.New("image exceeds worker memory budget") + +func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string { + extension := ".bin" + switch value.ContentType { + case "image/png": + extension = ".png" + case "image/jpeg": + extension = ".jpg" + case "image/webp": + extension = ".webp" + } + return path.Join( + "attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion), + name, value.SHA256+extension, + ) +} + +func (w *Worker) resolveProcessingFailure( + ctx context.Context, + delivery redisstream.Delivery, + lease model.AttachmentJobLease, + processingErr error, +) (string, error) { + code, permanent := processingError(processingErr) + if permanent || lease.Job.Attempt >= w.config.MaxAttempts { + if !permanent { + code = "retry_exhausted" + } + mutation, err := w.repository.FailAttachmentJob(ctx, lease, code) + if err != nil { + return "transient_error", fmt.Errorf("fail attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected failure result %q", mutation) + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + } + nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt)) + mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt) + if err != nil { + return "transient_error", fmt.Errorf("retry attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected retry result %q", mutation) + } + w.metrics.ObserveRetry() + return "retry", delivery.Ack(ctx) +} + +func processingError(err error) (string, bool) { + switch { + case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia): + return "invalid_image", true + case errors.Is(err, transform.ErrPayloadTooLarge): + return "image_too_large", true + case errors.Is(err, objectstore.ErrNotFound): + return "source_missing", true + case errors.Is(err, errWorkerCapacity): + return "worker_capacity", true + default: + return "processing_failed", false + } +} + +func (w *Worker) retryDelay(jobID string, attempt int) time.Duration { + delay := w.config.RetryBase + for step := 1; step < attempt && delay < w.config.RetryMax; step++ { + if delay > w.config.RetryMax/2 { + delay = w.config.RetryMax + break + } + delay *= 2 + } + if delay > w.config.RetryMax { + delay = w.config.RetryMax + } + var hash uint64 + for index := range jobID { + hash = hash*131 + uint64(jobID[index]) + } + jitterRange := min(delay/4, w.config.RetryMax-delay) + if jitterRange <= 0 { + return delay + } + jitter := time.Duration(hash % uint64(jitterRange)) + if delay > time.Duration(math.MaxInt64)-jitter { + return delay + } + return delay + jitter +} + +type nopMetrics struct{} + +func (nopMetrics) ObserveStarted() {} +func (nopMetrics) ObserveFinished(string, time.Duration) {} +func (nopMetrics) ObserveRetry() {} +func (nopMetrics) ObserveReclaimed(int) {} diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go new file mode 100644 index 0000000..7a573f1 --- /dev/null +++ b/internal/messenger/attachment/worker/worker_test.go @@ -0,0 +1,109 @@ +package worker + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" +) + +func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { + config := validConfig() + require.NoError(t, validateConfig(config)) + + config.MemoryBytes = 384 * 1024 * 1024 + require.ErrorContains(t, validateConfig(config), "memory must fit") +} + +func TestRetryDelayIsBounded(t *testing.T) { + worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}} + for attempt := 1; attempt <= 10; attempt++ { + delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt) + require.GreaterOrEqual(t, delay, time.Second) + require.LessOrEqual(t, delay, 4*time.Second) + } +} + +func TestProcessingErrorClassification(t *testing.T) { + tests := []struct { + err error + code string + permanent bool + }{ + {err: transform.ErrInvalidImage, code: "invalid_image", permanent: true}, + {err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true}, + {err: objectstore.ErrNotFound, code: "source_missing", permanent: true}, + {err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false}, + } + for _, test := range tests { + code, permanent := processingError(test.err) + require.Equal(t, test.code, code) + require.Equal(t, test.permanent, permanent) + } +} + +func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) { + var renewals atomic.Int32 + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + renewals.Add(1) + return true, nil + }} + config := validConfig() + config.HeartbeatInterval = 5 * time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }() + require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) +} + +func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) { + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + return false, nil + }} + config := validConfig() + config.HeartbeatInterval = time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + err := worker.maintainLease(ctx, cancel, testLease()) + require.ErrorIs(t, err, model.ErrStaleLease) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +type repositoryStub struct { + Repository + renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) +} + +func (s *repositoryStub) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + return s.renew(ctx, lease, duration) +} + +func testLease() model.AttachmentJobLease { + return model.AttachmentJobLease{Job: model.AttachmentJob{ + ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1", + }} +} + +func validConfig() Config { + return Config{ + WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second, + HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second, + ReclaimInterval: 5 * time.Second, + MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, + MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024, + } +} diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index bab14f6..f9250c8 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -6,6 +6,7 @@ import ( "mime" "net/http" "net/url" + "regexp" "strconv" "strings" "time" @@ -46,6 +47,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ ChatID: chatID, OwnerID: user.ID, Filename: filename, ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, + Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")), }) if err != nil { if errors.Is(err, model.ErrCapacity) { @@ -58,7 +60,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr } result = "success" s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes)) - writeJSON(w, http.StatusCreated, uploaded) + writeJSON(w, http.StatusAccepted, uploaded) return nil } @@ -108,20 +110,21 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE return authErr } query, err := url.ParseQuery(r.URL.RawQuery) - if err != nil || !originalVariant(query) { + variant, valid := attachmentVariant(query) + if err != nil || !valid { return invalidRequest() } attachmentID, apiErr := attachmentIDFromRequest(r) if apiErr != nil { return apiErr } - content, err := s.attachments.Content(r.Context(), attachmentID, user.ID) + content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant) if err != nil { return attachmentError(err) } defer func() { _ = content.Body.Close() }() - w.Header().Set("Content-Type", content.Attachment.ContentType) - w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10)) + w.Header().Set("Content-Type", content.Variant.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10)) w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, content.Body); err != nil { @@ -130,11 +133,35 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE return nil } -func originalVariant(query url.Values) bool { +func attachmentVariant(query url.Values) (string, bool) { if len(query) == 0 { - return true + return "original", true } - return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original" + if len(query) != 1 || len(query["variant"]) != 1 { + return "", false + } + switch query["variant"][0] { + case "original", "thumbnail", "crop", "optimized": + return query["variant"][0], true + default: + return "", false + } +} + +var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +func traceparent(value string) string { + if traceparentPattern.MatchString(value) { + return value + } + return "" +} + +func tracestate(value string) string { + if len(value) <= 512 { + return value + } + return "" } func attachmentIDFromRequest(r *http.Request) (string, *apiError) { @@ -158,6 +185,8 @@ func attachmentError(err error) *apiError { return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} case errors.Is(err, model.ErrCapacity): return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + case errors.Is(err, model.ErrConflict): + return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"} default: return unavailable() } diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go index 5fd83eb..055b80b 100644 --- a/internal/messenger/httpapi/messenger_interface.go +++ b/internal/messenger/httpapi/messenger_interface.go @@ -28,7 +28,7 @@ type MessageRepository interface { type AttachmentService interface { Upload(context.Context, attachment.Upload) (model.Attachment, error) Metadata(context.Context, string, string) (model.Attachment, error) - Content(context.Context, string, string) (attachment.Content, error) + Content(context.Context, string, string, string) (attachment.Content, error) } type HealthChecker interface { diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go index 1dae064..f1b058e 100644 --- a/internal/messenger/httpapi/metrics.go +++ b/internal/messenger/httpapi/metrics.go @@ -11,6 +11,11 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +var ( + requestDurationBuckets = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30} + uploadDurationBuckets = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60} +) + type metrics struct { requests *prometheus.CounterVec duration *prometheus.HistogramVec @@ -36,7 +41,7 @@ func newMetrics() *metrics { Subsystem: "http", Name: "request_duration_seconds", Help: "HTTP request duration in seconds.", - Buckets: prometheus.DefBuckets, + Buckets: requestDurationBuckets, }, []string{"method", "route"}), inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "messenger", @@ -59,7 +64,7 @@ func newMetrics() *metrics { Namespace: "messenger", Name: "attachment_upload_duration_seconds", Help: "Attachment upload duration in seconds.", - Buckets: prometheus.DefBuckets, + Buckets: uploadDurationBuckets, }), attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "messenger", diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index 8199a0d..044121d 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -87,8 +87,12 @@ func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) { return s.uploaded, s.err } -func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) { - return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err +func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) { + return attachment.Content{ + Attachment: s.uploaded, + Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes}, + Body: io.NopCloser(strings.NewReader("content")), + }, s.err } type sessionStub struct { diff --git a/internal/model/errors.go b/internal/model/errors.go index dbe09f0..83105b2 100644 --- a/internal/model/errors.go +++ b/internal/model/errors.go @@ -15,4 +15,10 @@ var ( ErrPayloadTooLarge = errors.New("payload too large") ErrCapacity = errors.New("capacity exhausted") ErrUnavailable = errors.New("dependency unavailable") + ErrStaleLease = errors.New("stale lease") + ErrStaleAttempt = errors.New("stale attempt") + ErrStalePipeline = errors.New("stale pipeline") + ErrJobBusy = errors.New("job is already leased") + ErrJobComplete = errors.New("job is already complete") + ErrJobNotDue = errors.New("job is not due") ) diff --git a/internal/model/model.go b/internal/model/model.go index bc71b97..83f7995 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -6,9 +6,21 @@ import ( ) const ( - AttachmentPending = "pending" - AttachmentReady = "ready" - AttachmentDeleting = "deleting" + AttachmentPending = "pending" + AttachmentQueued = "queued" + AttachmentProcessing = "processing" + AttachmentReady = "ready" + AttachmentFailed = "failed" + AttachmentDeleting = "deleting" + + AttachmentJobQueued = "queued" + AttachmentJobProcessing = "processing" + AttachmentJobReady = "ready" + AttachmentJobFailed = "failed" + + JobMutationApplied JobMutationResult = "applied" + JobMutationAlreadyComplete JobMutationResult = "already_complete" + JobMutationStale JobMutationResult = "stale" ) const timestampLayout = "2006-01-02T15:04:05.000000Z" @@ -42,21 +54,23 @@ type Message struct { } type Attachment struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []Variant `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ObjectKey string `json:"-"` - ReadyAt *time.Time `json:"-"` - Referenced bool `json:"-"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + StagingKey string `json:"-"` + ReadyAt *time.Time `json:"-"` + Referenced bool `json:"-"` + PipelineVersion int `json:"-"` } type StoredObject struct { @@ -66,12 +80,68 @@ type StoredObject struct { } type Variant struct { - Name string `json:"name"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Width int `json:"width"` - Height int `json:"height"` + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Width int `json:"width"` + Height int `json:"height"` + ObjectKey string `json:"-"` + PipelineVersion int `json:"-"` +} + +type AttachmentJob struct { + ID string + AttachmentID string + PipelineVersion int + Attempt int + Status string + AvailableAt time.Time + LeaseToken string + LeaseOwner string + LeaseUntil *time.Time + ErrorCode *string + CreatedAt time.Time + UpdatedAt time.Time +} + +type AttachmentJobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +type AttachmentJobLease struct { + Job AttachmentJob + Attachment Attachment +} + +type OutboxEvent struct { + ID string + Topic string + PartitionKey string + Payload json.RawMessage + JobID string + Attempt int + AvailableAt time.Time + PublishTries int + LeaseToken string + LeaseUntil *time.Time + LastErrorCode *string + CreatedAt time.Time +} + +type JobMutationResult string + +type RecoveryResult struct { + Requeued int + Failed int } type Position struct { diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go new file mode 100644 index 0000000..22aa75d --- /dev/null +++ b/internal/outbox/dispatcher.go @@ -0,0 +1,203 @@ +package outbox + +import ( + "context" + "errors" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +var errLeaseLost = errors.New("outbox lease lost") + +type Store interface { + ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) + MarkOutboxPublished(context.Context, string, string) (bool, error) + ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error) + RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) + DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error) +} + +type Publisher interface { + PublishConfirmed(context.Context, []byte) (string, error) +} + +type Metrics interface { + ObservePublished() + ObservePublishFailure() + ObserveRecovered(int) +} + +type Config struct { + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int +} + +type Dispatcher struct { + store Store + publisher Publisher + metrics Metrics + config Config + now func() time.Time +} + +func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) { + if store == nil { + return nil, errors.New("outbox store is required") + } + if publisher == nil { + return nil, errors.New("outbox publisher is required") + } + if err := config.validate(); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil +} + +func (c Config) validate() error { + switch { + case c.BatchSize <= 0: + return errors.New("outbox batch size must be positive") + case c.LeaseDuration <= 0: + return errors.New("outbox lease duration must be positive") + case c.PollInterval <= 0: + return errors.New("outbox poll interval must be positive") + case c.RecoveryInterval <= 0: + return errors.New("outbox recovery interval must be positive") + case c.RetryDelay <= 0: + return errors.New("outbox retry delay must be positive") + case c.Retention <= 0: + return errors.New("outbox retention must be positive") + case c.CleanupInterval <= 0: + return errors.New("outbox cleanup interval must be positive") + case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000: + return errors.New("outbox cleanup batch size must be between 1 and 1000") + default: + return nil + } +} + +func (d *Dispatcher) Run(ctx context.Context) error { + if ctx.Err() != nil { + return nil + } + + pollTicker := time.NewTicker(d.config.PollInterval) + recoveryTicker := time.NewTicker(d.config.RecoveryInterval) + cleanupTicker := time.NewTicker(d.config.CleanupInterval) + defer pollTicker.Stop() + defer recoveryTicker.Stop() + defer cleanupTicker.Stop() + + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + for { + if _, err := d.DispatchOnce(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + + select { + case <-ctx.Done(): + return nil + case <-pollTicker.C: + case <-recoveryTicker.C: + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + case <-cleanupTicker.C: + if _, err := d.CleanupPublished(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + } + } +} + +func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) { + deleted, err := d.store.DeletePublishedOutboxEvents( + ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize, + ) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return deleted, nil +} + +func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { + events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration) + if err != nil { + return 0, fmt.Errorf("claim due outbox events: %w", err) + } + + published := 0 + for _, event := range events { + if err := ctx.Err(); err != nil { + return published, nil + } + _, err := d.publisher.PublishConfirmed(ctx, event.Payload) + if err != nil { + if ctx.Err() != nil { + return published, nil + } + d.metrics.ObservePublishFailure() + nextAttemptAt := d.now().Add(d.config.RetryDelay) + released, releaseErr := d.store.ReleaseOutboxEvent( + ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, + ) + if releaseErr != nil { + return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr) + } + if !released { + return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost) + } + continue + } + marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken) + if err != nil { + return published, fmt.Errorf("mark outbox event published: %w", err) + } + if !marked { + return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost) + } + d.metrics.ObservePublished() + published++ + } + return published, nil +} + +func (d *Dispatcher) RecoverExpired(ctx context.Context) error { + recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now()) + if err != nil { + return fmt.Errorf("recover expired outbox leases: %w", err) + } + d.metrics.ObserveRecovered(recovered) + return nil +} + +type nopMetrics struct{} + +func (nopMetrics) ObservePublished() {} +func (nopMetrics) ObservePublishFailure() {} +func (nopMetrics) ObserveRecovered(int) {} diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go new file mode 100644 index 0000000..206790d --- /dev/null +++ b/internal/outbox/dispatcher_test.go @@ -0,0 +1,207 @@ +package outbox + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "github.com/stretchr/testify/require" +) + +func TestDispatcherPublishesAndMarksEvent(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, []string{"event-1:lease-1"}, store.marked) +} + +func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) { + event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()} + store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + _, err := dispatcher.DispatchOnce(context.Background()) + require.Error(t, err) + require.Equal(t, 1, publisher.calls) + + store.markErr = nil + store.events[0].LeaseToken = "lease-2" + publisher.messageID = "2-0" + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, 2, publisher.calls) + require.JSONEq(t, string(event.Payload), string(publisher.payloads[0])) + require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1])) + require.Equal(t, []string{"event-1:lease-2"}, store.marked) +} + +func TestDispatcherReleasesPublishFailure(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{err: errors.New("redis unavailable")} + dispatcher := newTestDispatcher(t, store, publisher) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Zero(t, published) + require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released) + require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt) +} + +func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) { + store := &storeStub{ + events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}}, + markLost: true, + } + dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"}) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.ErrorIs(t, err, errLeaseLost) + require.Zero(t, published) +} + +func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) { + store := &storeStub{deleted: 2} + dispatcher := newTestDispatcher(t, store, &publisherStub{}) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + deleted, err := dispatcher.CleanupPublished(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, deleted) + require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore) + require.Equal(t, 100, store.cleanupLimit) +} + +func TestDispatcherStopsAfterCancellation(t *testing.T) { + store := &storeStub{} + publisher := &publisherStub{} + dispatcher := newTestDispatcher(t, store, publisher) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { done <- dispatcher.Run(ctx) }() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("dispatcher did not stop after cancellation") + } +} + +func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher { + t.Helper() + dispatcher, err := New(store, publisher, nil, Config{ + BatchSize: 10, + LeaseDuration: time.Minute, + PollInterval: time.Millisecond, + RecoveryInterval: time.Second, + RetryDelay: time.Second, + Retention: 24 * time.Hour, + CleanupInterval: time.Hour, + CleanupBatchSize: 100, + }) + require.NoError(t, err) + return dispatcher +} + +func testPayload() json.RawMessage { + return json.RawMessage(`{ + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-10-22T12:00:00Z" + }`) +} + +type storeStub struct { + mu sync.Mutex + events []model.OutboxEvent + markErr error + markLost bool + marked []string + released string + nextAttemptAt time.Time + recoveredCalls int + deleted int + publishedBefore time.Time + cleanupLimit int +} + +func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]model.OutboxEvent(nil), s.events...), nil +} + +func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.markErr != nil { + return false, s.markErr + } + if s.markLost { + return false, nil + } + s.marked = append(s.marked, id+":"+leaseToken) + return true, nil +} + +func (s *storeStub) ReleaseOutboxEvent( + _ context.Context, + id string, + leaseToken string, + code string, + nextAttemptAt time.Time, +) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.released = id + ":" + leaseToken + ":" + code + s.nextAttemptAt = nextAttemptAt + return true, nil +} + +func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.recoveredCalls++ + return 0, nil +} + +func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishedBefore = before + s.cleanupLimit = limit + return s.deleted, nil +} + +type publisherStub struct { + mu sync.Mutex + messageID string + err error + calls int + payloads [][]byte +} + +func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.calls++ + p.payloads = append(p.payloads, append([]byte(nil), payload...)) + return p.messageID, p.err +} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go new file mode 100644 index 0000000..3bcfb39 --- /dev/null +++ b/internal/postgres/attachment_jobs.go @@ -0,0 +1,563 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +const attachmentJobTopic = "attachment.jobs" + +const attachmentJobColumns = ` + id, attachment_id, pipeline_version, attempt, status, available_at, + lease_token, lease_owner, lease_until, error_code, created_at, updated_at` + +func (s *Store) QueueAttachmentWithJobAndOutbox( + ctx context.Context, + attachmentID string, + stagingKey string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 1 { + return model.Attachment{}, model.ErrInvalidInput + } + if job.Attempt == 0 { + job.Attempt = 1 + } + if job.Attempt != 1 { + return model.Attachment{}, model.ErrInvalidInput + } + now := time.Now().UTC() + if job.CreatedAt.IsZero() { + job.CreatedAt = now + } + if job.AvailableAt.IsZero() { + job.AvailableAt = now + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + query := ` + UPDATE attachments + SET status = 'queued', pipeline_version = $3, staging_key = $2, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + var attachment model.Attachment + err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job) + } + if err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment: %w", err) + } + + job.AttachmentID = attachmentID + job.Status = model.AttachmentJobQueued + job.UpdatedAt = job.CreatedAt + if err := insertAttachmentJob(ctx, tx, job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) QueueAttachmentPipelineVersion( + ctx context.Context, + attachmentID string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 2 { + return model.Attachment{}, model.ErrInvalidInput + } + job.Attempt = 1 + now := time.Now().UTC() + job.CreatedAt = now + job.AvailableAt = now + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err) + } + if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 { + return model.Attachment{}, ErrConflict + } + job.AttachmentID = attachment.ID + job.Status = model.AttachmentJobQueued + job.UpdatedAt = now + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', pipeline_version = $2, staging_key = object_key, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 + RETURNING ` + attachmentColumns + if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err) + } + if err := insertAttachmentJob(ctx, tx, job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) alreadyQueuedAttachment( + ctx context.Context, + tx pgx.Tx, + attachmentID string, + stagingKey string, + job model.AttachmentJob, +) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN attachment_jobs j + ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready') + AND j.id = $3 AND j.pipeline_version = $4` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrConflict + } + return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error { + const query = ` + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)` + if _, err := tx.Exec( + ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt, + ); err != nil { + return fmt.Errorf("insert attachment job: %w", err) + } + return nil +} + +func insertAttachmentJobOutbox( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + traceparent string, + tracestate string, +) error { + payload, err := json.Marshal(model.AttachmentJobMessage{ + SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1", + AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, CreatedAt: job.CreatedAt, + Traceparent: traceparent, Tracestate: tracestate, + }) + if err != nil { + return fmt.Errorf("marshal attachment job event: %w", err) + } + const query = ` + INSERT INTO outbox_events ( + id, topic, partition_key, payload, job_id, attempt, available_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload, + job.ID, job.Attempt, job.AvailableAt, + ); err != nil { + return fmt.Errorf("insert attachment job outbox event: %w", err) + } + return nil +} + +func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) { + const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1` + var job model.AttachmentJob + if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrNotFound + } + return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err) + } + return job, nil +} + +func (s *Store) ClaimAttachmentJob( + ctx context.Context, + jobID string, + attachmentID string, + pipelineVersion int, + attempt int, + workerID string, + leaseDuration time.Duration, +) (model.AttachmentJobLease, error) { + if leaseDuration <= 0 || workerID == "" { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJobLease{}, err + } + if attachment.ID != attachmentID || job.AttachmentID != attachmentID { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady { + return model.AttachmentJobLease{}, model.ErrJobComplete + } + if job.Attempt != attempt { + return model.AttachmentJobLease{}, model.ErrStaleAttempt + } + if job.Status == model.AttachmentJobProcessing { + return model.AttachmentJobLease{}, model.ErrJobBusy + } + if job.Status != model.AttachmentJobQueued { + return model.AttachmentJobLease{}, ErrConflict + } + if job.AvailableAt.After(time.Now()) { + return model.AttachmentJobLease{}, model.ErrJobNotDue + } + + job.LeaseToken = uuid.NewString() + job.LeaseOwner = workerID + leaseUntil := time.Now().UTC().Add(leaseDuration) + job.LeaseUntil = &leaseUntil + job.Status = model.AttachmentJobProcessing + const updateJob = ` + UPDATE attachment_jobs + SET status = 'processing', lease_token = $2, lease_owner = $3, + lease_until = $4, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err) + } + const updateAttachment = ` + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'` + result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err) + } + if result.RowsAffected() != 1 { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err) + } + attachment.Status = model.AttachmentProcessing + attachment.ErrorCode = nil + return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil +} + +func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) { + query := ` + SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + ` + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = $1 + FOR UPDATE OF j, a` + var job model.AttachmentJob + var attachment model.Attachment + var leaseToken, leaseOwner *string + var stagingKey *string + err := tx.QueryRow(ctx, query, jobID).Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, model.Attachment{}, ErrNotFound + } + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err) + } + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return job, attachment, nil +} + +func prefixedAttachmentJobColumns(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+ + "%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+ + "%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at", + alias, + ) +} + +func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error { + var leaseToken, leaseOwner *string + err := row.Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + ) + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + return err +} + +func (s *Store) CompleteAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + variants []model.Variant, +) (model.JobMutationResult, error) { + if !validCompletedVariants(variants, lease.Job.PipelineVersion) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin complete attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const deleteVariants = ` + DELETE FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("replace attachment variants: %w", err) + } + + var originalKey string + const insertVariant = ` + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + sha256, width, height, object_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + for _, variant := range variants { + if variant.Name == "original" { + originalKey = variant.ObjectKey + } + if _, err := tx.Exec( + ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name, + variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width, + variant.Height, variant.ObjectKey, + ); err != nil { + return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err) + } + } + + const completeJob = ` + UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil { + return "", fmt.Errorf("complete attachment job: %w", err) + } + const completeAttachment = ` + UPDATE attachments + SET status = 'ready', object_key = $3, ready_at = clock_timestamp(), + staging_key = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil { + return "", fmt.Errorf("complete attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit completed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func validCompletedVariants(variants []model.Variant, pipelineVersion int) bool { + if len(variants) != 4 || pipelineVersion < 1 { + return false + } + required := map[string]struct{}{ + "original": {}, "thumbnail": {}, "crop": {}, "optimized": {}, + } + for _, variant := range variants { + if variant.ObjectKey == "" || variant.PipelineVersion != pipelineVersion { + return false + } + if _, exists := required[variant.Name]; !exists { + return false + } + delete(required, variant.Name) + } + return len(required) == 0 +} + +func (s *Store) RetryAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, + nextAttemptAt time.Time, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin retry attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = nextAttemptAt.UTC() + job.ErrorCode = &errorCode + const retryJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $3, available_at = $4, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $5, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec( + ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode, + ); err != nil { + return "", fmt.Errorf("retry attachment job: %w", err) + } + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("requeue attachment: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return "", err + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit attachment job retry: %w", err) + } + return model.JobMutationApplied, nil +} + +func (s *Store) FailAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin fail attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $3, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil { + return "", fmt.Errorf("fail attachment job: %w", err) + } + const failAttachment = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return "", fmt.Errorf("fail attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit failed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func lockJobMutation( + ctx context.Context, + tx pgx.Tx, + lease model.AttachmentJobLease, +) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) { + job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID) + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, "", err + } + if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion { + return job, attachment, model.JobMutationStale, nil + } + if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady { + return job, attachment, model.JobMutationAlreadyComplete, nil + } + if job.PipelineVersion != lease.Job.PipelineVersion || + job.Attempt != lease.Job.Attempt || + job.Status != model.AttachmentJobProcessing || + job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken || + job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) { + return job, attachment, model.JobMutationStale, nil + } + return job, attachment, model.JobMutationApplied, nil +} + +func validErrorCode(errorCode string) bool { + return len(errorCode) >= 1 && len(errorCode) <= 64 +} diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go new file mode 100644 index 0000000..b6883eb --- /dev/null +++ b/internal/postgres/attachment_lease.go @@ -0,0 +1,32 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE attachment_jobs + SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'), + updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2 AND lease_owner = $3 + AND status = 'processing' AND lease_until > clock_timestamp()` + result, err := s.pool.Exec( + ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(), + ) + if err != nil { + return false, fmt.Errorf("renew attachment job lease: %w", err) + } + return result.RowsAffected() == 1, nil +} diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go new file mode 100644 index 0000000..d7d84ca --- /dev/null +++ b/internal/postgres/attachment_recovery.go @@ -0,0 +1,210 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) RecoverExpiredAttachmentJobs( + ctx context.Context, + limit int, + maxAttempts int, + retryAt time.Time, +) (model.RecoveryResult, error) { + if limit < 1 || limit > 1000 || maxAttempts < 1 { + return model.RecoveryResult{}, model.ErrInvalidInput + } + if retryAt.IsZero() { + retryAt = time.Now().UTC() + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectExpired = ` + SELECT id + FROM attachment_jobs + WHERE status = 'processing' AND lease_until <= clock_timestamp() + ORDER BY lease_until, id + LIMIT $1 + FOR UPDATE SKIP LOCKED` + rows, err := tx.Query(ctx, selectExpired, limit) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err) + } + jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err) + } + + var recovered model.RecoveryResult + for _, jobID := range jobIDs { + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.RecoveryResult{}, err + } + if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) { + continue + } + if job.PipelineVersion != attachment.PipelineVersion { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + if job.Attempt >= maxAttempts { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = retryAt.UTC() + const requeueJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = 'lease_expired', updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err) + } + const requeueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return model.RecoveryResult{}, err + } + recovered.Requeued++ + } + if err := tx.Commit(ctx); err != nil { + return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err) + } + return recovered, nil +} + +func markExpiredJobFailed( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + attachment model.Attachment, + errorCode string, + failAttachment bool, +) error { + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $2, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil { + return fmt.Errorf("fail expired attachment job: %w", err) + } + if !failAttachment { + return nil + } + const failAttachmentQuery = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return fmt.Errorf("fail attachment after expired lease: %w", err) + } + return nil +} + +func (s *Store) ReplayFailedAttachment( + ctx context.Context, + attachmentID string, + actor string, + reason string, +) (model.AttachmentJob, error) { + if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 { + return model.AttachmentJob{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectJob = ` + SELECT j.id + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed' + FOR UPDATE OF j, a` + var jobID string + if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrConflict + } + return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err) + } + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJob{}, err + } + previousAttempt := job.Attempt + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = time.Now().UTC() + job.ErrorCode = nil + job.LeaseToken = "" + job.LeaseOwner = "" + job.LeaseUntil = nil + const replayJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err) + } + const replayAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err) + } + const insertAudit = ` + INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID, + previousAttempt, job.Attempt, actor, reason, + ); err != nil { + return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err) + } + eventJob := job + eventJob.CreatedAt = time.Now().UTC() + if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + return model.AttachmentJob{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err) + } + return job, nil +} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index 8728ae3..e7c2531 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -13,7 +13,7 @@ import ( const attachmentColumns = ` id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, status, created_at, updated_at, ready_at` + object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key` type rowScanner interface { Scan(...any) error @@ -74,7 +74,7 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri SELECT ` + attachmentColumnsWithAlias("a") + ` FROM attachments a JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 - WHERE a.id = $1 AND a.status = 'ready'` + WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')` var attachment model.Attachment err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment) @@ -84,9 +84,68 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri if err != nil { return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) } + if attachment.Status == model.AttachmentReady { + attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion) + if err != nil { + return model.Attachment{}, err + } + } return publicAttachment(attachment), nil } +func (s *Store) AttachmentVariantForUser( + ctx context.Context, + attachmentID string, + userID string, + variantName string, +) (model.Attachment, model.Variant, error) { + attachment, err := s.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, model.Variant{}, err + } + if attachment.Status != model.AttachmentReady { + return model.Attachment{}, model.Variant{}, ErrConflict + } + for _, variant := range attachment.Variants { + if variant.Name == variantName { + return attachment, variant, nil + } + } + if variantName == "original" && attachment.ObjectKey != "" { + return attachment, model.Variant{ + Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey, + PipelineVersion: attachment.PipelineVersion, + }, nil + } + return model.Attachment{}, model.Variant{}, ErrNotFound +} + +func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) { + const query = ` + SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY CASE name + WHEN 'original' THEN 0 + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END` + rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion) + if err != nil { + return nil, fmt.Errorf("list attachment variants: %w", err) + } + defer rows.Close() + + variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant]) + if err != nil { + return nil, fmt.Errorf("scan attachment variants: %w", err) + } + return variants, nil +} + func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2` @@ -102,7 +161,12 @@ func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey } func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { - const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)` + const query = ` + SELECT EXISTS ( + SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1 + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = $1 + )` var exists bool if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil { return false, fmt.Errorf("check attachment by object key: %w", err) @@ -110,16 +174,51 @@ func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey strin return exists, nil } +func (s *Store) AttachmentObjectKeys(ctx context.Context, attachmentID string) ([]string, error) { + const query = ` + SELECT object_key + FROM ( + SELECT object_key + FROM attachments + WHERE id = $1 AND object_key <> '' + UNION + SELECT object_key + FROM attachment_variants + WHERE attachment_id = $1 + ) objects + ORDER BY object_key` + rows, err := s.pool.Query(ctx, query, attachmentID) + if err != nil { + return nil, fmt.Errorf("list attachment object keys: %w", err) + } + defer rows.Close() + + keys, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("scan attachment object keys: %w", err) + } + return keys, nil +} + func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { if limit < 1 || limit > 1000 { return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") } query := ` SELECT ` + attachmentColumnsWithAlias("a") + `, - EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) - FROM attachments a - WHERE a.object_key > $1 - ORDER BY a.object_key + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id), + objects.object_key + FROM ( + SELECT id AS attachment_id, object_key + FROM attachments + WHERE object_key <> '' + UNION + SELECT attachment_id, object_key + FROM attachment_variants + ) objects + JOIN attachments a ON a.id = objects.attachment_id + WHERE objects.object_key > $1 + ORDER BY objects.object_key LIMIT $2` rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) if err != nil { @@ -129,7 +228,9 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { var attachment model.Attachment - err := scanAttachmentWithReference(row, &attachment) + var objectKey string + err := scanAttachmentWithReferenceAndObjectKey(row, &attachment, &objectKey) + attachment.ObjectKey = objectKey return publicAttachment(attachment), err })) if err != nil { @@ -143,13 +244,38 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi return attachments, next, nil } -func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { - return row.Scan( +func scanAttachmentWithReferenceAndObjectKey( + row rowScanner, + attachment *model.Attachment, + objectKey *string, +) error { + var stagingKey *string + err := row.Scan( &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + &attachment.Referenced, objectKey, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, &attachment.Referenced, ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err } func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { @@ -232,23 +358,36 @@ func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID str } func scanAttachment(row rowScanner, attachment *model.Attachment) error { - return row.Scan( + var stagingKey *string + err := row.Scan( &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err } func attachmentColumnsWithAlias(alias string) string { return fmt.Sprintf( "%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+ "%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+ - "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at", + "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+ + "%[1]s.error_code, %[1]s.staging_key", alias, ) } func publicAttachment(attachment model.Attachment) model.Attachment { - attachment.Variants = []model.Variant{} + variants := make([]model.Variant, 0, len(attachment.Variants)) + for _, variant := range attachment.Variants { + if variant.Name != "original" { + variants = append(variants, variant) + } + } + attachment.Variants = variants return attachment } diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index 0e56e16..e4daccb 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -8,4 +8,10 @@ var ( ErrInvalidMembers = model.ErrInvalidMembers ErrInvalidCursor = model.ErrInvalidCursor ErrIdempotencyConflict = model.ErrIdempotencyConflict + ErrStaleLease = model.ErrStaleLease + ErrStaleAttempt = model.ErrStaleAttempt + ErrStalePipeline = model.ErrStalePipeline + ErrJobBusy = model.ErrJobBusy + ErrJobComplete = model.ErrJobComplete + ErrJobNotDue = model.ErrJobNotDue ) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index 49f576e..a8168f5 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -51,13 +51,16 @@ func (s *Store) CreateMessage( if !visible { return model.Message{}, false, ErrNotFound } - validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) + validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) if err != nil { return model.Message{}, false, err } if !validAttachments { return model.Message{}, false, ErrNotFound } + if failedAttachment { + return model.Message{}, false, ErrConflict + } if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { return model.Message{}, false, err } @@ -92,28 +95,35 @@ func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, return sha256.Sum256(payload), nil } -func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) { +func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) { if len(attachmentIDs) == 0 { - return true, nil + return true, false, nil } const query = ` - SELECT id + SELECT id, status FROM attachments - WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready' + WHERE chat_id = $1 AND id::text = ANY($2) + AND status IN ('queued', 'processing', 'ready', 'failed') FOR UPDATE` rows, err := tx.Query(ctx, query, chatID, attachmentIDs) if err != nil { - return false, fmt.Errorf("validate message attachments: %w", err) + return false, false, fmt.Errorf("validate message attachments: %w", err) } defer rows.Close() - count := 0 + var count int + var failed bool for rows.Next() { + var id, status string + if err := rows.Scan(&id, &status); err != nil { + return false, false, fmt.Errorf("scan message attachment: %w", err) + } count++ + failed = failed || status == model.AttachmentFailed } if err := rows.Err(); err != nil { - return false, fmt.Errorf("validate message attachments: %w", err) + return false, false, fmt.Errorf("validate message attachments: %w", err) } - return count == len(attachmentIDs), nil + return count == len(attachmentIDs), failed, nil } func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go new file mode 100644 index 0000000..9eda865 --- /dev/null +++ b/internal/postgres/outbox.go @@ -0,0 +1,136 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) ClaimOutboxEvents( + ctx context.Context, + limit int, + leaseDuration time.Duration, +) ([]model.OutboxEvent, error) { + if limit < 1 || limit > 1000 || leaseDuration <= 0 { + return nil, model.ErrInvalidInput + } + claimToken := uuid.NewString() + query := ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at IS NULL + AND available_at <= clock_timestamp() + AND (claim_until IS NULL OR claim_until <= clock_timestamp()) + ORDER BY available_at, id + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE outbox_events event + SET claim_token = $2, + claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond', + publish_tries = event.publish_tries + 1 + FROM candidates + WHERE event.id = candidates.id + RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id, + event.attempt, event.available_at, event.publish_tries, event.claim_token, + event.claim_until, event.last_error_code, event.created_at` + rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds()) + if err != nil { + return nil, fmt.Errorf("claim outbox events: %w", err) + } + defer rows.Close() + + events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) { + var event model.OutboxEvent + err := row.Scan( + &event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID, + &event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken, + &event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt, + ) + return event, err + })) + if err != nil { + return nil, fmt.Errorf("scan claimed outbox events: %w", err) + } + return events, nil +} + +func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) { + const query = ` + UPDATE outbox_events + SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL, + last_error_code = NULL + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken) + if err != nil { + return false, fmt.Errorf("mark outbox event published: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) ReleaseOutboxEvent( + ctx context.Context, + eventID string, + leaseToken string, + errorCode string, + nextAttemptAt time.Time, +) (bool, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE outbox_events + SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4 + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode) + if err != nil { + return false, fmt.Errorf("release outbox event: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) { + if now.IsZero() { + now = time.Now().UTC() + } + const query = ` + UPDATE outbox_events + SET claim_token = NULL, claim_until = NULL + WHERE published_at IS NULL AND claim_until <= $1` + result, err := s.pool.Exec(ctx, query, now.UTC()) + if err != nil { + return 0, fmt.Errorf("recover expired outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} + +func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) { + if publishedBefore.IsZero() || limit < 1 || limit > 1000 { + return 0, model.ErrInvalidInput + } + const query = ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at < $1 + ORDER BY published_at, id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM outbox_events event + USING candidates + WHERE event.id = candidates.id` + result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 5974d28..e478d9b 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -2,7 +2,9 @@ package postgres_test import ( "context" + "encoding/json" "errors" + "fmt" "os" "strings" "sync" @@ -159,6 +161,224 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { s.False(deleted) } +func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + s.Equal(model.AttachmentQueued, attachment.Status) + + events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute) + s.Require().NoError(err) + s.Require().NotEmpty(events) + var event model.OutboxEvent + for _, candidate := range events { + if candidate.JobID == job.ID { + event = candidate + break + } + } + s.Require().NotEmpty(event.ID) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(event.Payload, &message)) + s.Equal(job.ID, message.JobID) + s.Equal(attachment.ID, message.AttachmentID) + s.Equal(1, message.Attempt) + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + + _, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "queued attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute, + ) + s.ErrorIs(err, model.ErrInvalidInput) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + s.Equal(model.AttachmentProcessing, lease.Attachment.Status) + s.NotEmpty(lease.Attachment.StagingKey) + + variants := completedVariants(attachment.ID, 1) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationAlreadyComplete, result) + + metadata, variant, err := s.store.AttachmentVariantForUser( + context.Background(), attachment.ID, owner.ID, "thumbnail", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, metadata.Status) + s.Len(metadata.Variants, 3) + s.Equal(variants[1].ObjectKey, variant.ObjectKey) + listedKeys := make(map[string]struct{}) + var after string + for { + listed, next, err := s.store.ListAttachments(context.Background(), after, 1000) + s.Require().NoError(err) + for _, candidate := range listed { + if candidate.ID == attachment.ID { + listedKeys[candidate.ObjectKey] = struct{}{} + } + } + if next == "" { + break + } + after = next + } + for _, candidate := range variants { + s.Contains(listedKeys, candidate.ObjectKey) + } + + v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2} + queuedV2, err := s.store.QueueAttachmentPipelineVersion( + context.Background(), attachment.ID, v2, "", "", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentQueued, queuedV2.Status) + s.Equal(2, queuedV2.PipelineVersion) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + _, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + _, secondJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + events, err := s.store.ClaimOutboxEvents(context.Background(), 1000, time.Minute) + s.Require().NoError(err) + + eventIDs := make([]string, 0, 2) + for _, event := range events { + if event.JobID != firstJob.ID && event.JobID != secondJob.ID { + continue + } + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + eventIDs = append(eventIDs, event.ID) + } + s.Require().Len(eventIDs, 2) + _, err = s.pool.Exec( + context.Background(), + `UPDATE outbox_events SET published_at = clock_timestamp() - INTERVAL '2 days' WHERE id IN ($1, $2)`, + eventIDs[0], eventIDs[1], + ) + s.Require().NoError(err) + + deleted, err := s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + deleted, err = s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + + var remaining int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM outbox_events WHERE id IN ($1, $2)`, eventIDs[0], eventIDs[1], + ).Scan(&remaining) + s.Require().NoError(err) + s.Zero(remaining) +} + +func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image") + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + + _, _, err = s.store.CreateMessage( + context.Background(), chatID, owner.ID, "failed attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrConflict) + metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentFailed, metadata.Status) + s.Require().NotNil(metadata.ErrorCode) + s.Equal("invalid_image", *metadata.ErrorCode) + + replayed, err := s.store.ReplayFailedAttachment( + context.Background(), attachment.ID, "operator@example", "input repaired", + ) + s.Require().NoError(err) + s.Equal(2, replayed.Attempt) + s.Equal(model.AttachmentJobQueued, replayed.Status) + + var auditCount int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID, + ).Scan(&auditCount) + s.Require().NoError(err) + s.Equal(1, auditCount) +} + +func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now()) + s.Require().NoError(err) + s.Equal(1, recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(2, current.Attempt) + s.Equal(model.AttachmentJobQueued, current.Status) + + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1)) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + + complete := completedVariants(attachment.ID, job.PipelineVersion) + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, complete[:3]) + s.ErrorIs(err, model.ErrInvalidInput) + + duplicate := append([]model.Variant(nil), complete...) + duplicate[3].Name = duplicate[2].Name + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, duplicate) + s.ErrorIs(err, model.ErrInvalidInput) + + mismatched := append([]model.Variant(nil), complete...) + mismatched[3].PipelineVersion++ + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, mismatched) + s.ErrorIs(err, model.ErrInvalidInput) +} + func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { owner, chatID := createOwnerAndChat(s.T(), s.store) pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ @@ -344,6 +564,58 @@ func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string return owner, chat.ID } +func createQueuedAttachment( + t *testing.T, + store *postgres.Store, + ownerID string, + chatID string, +) (model.Attachment, model.AttachmentJob) { + t.Helper() + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: stagingKey, + }) + if err != nil { + t.Fatalf("CreatePendingAttachment() error = %v", err) + } + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1, + } + queued, err := store.QueueAttachmentWithJobAndOutbox( + context.Background(), pending.ID, stagingKey, job, "", "", + ) + if err != nil { + t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err) + } + return queued, job +} + +func completedVariants(attachmentID string, pipelineVersion int) []model.Variant { + types := []struct { + name string + contentType string + extension string + }{ + {name: "original", contentType: "image/png", extension: "png"}, + {name: "thumbnail", contentType: "image/jpeg", extension: "jpg"}, + {name: "crop", contentType: "image/jpeg", extension: "jpg"}, + {name: "optimized", contentType: "image/webp", extension: "webp"}, + } + variants := make([]model.Variant, 0, len(types)) + for index, variant := range types { + variants = append(variants, model.Variant{ + Name: variant.name, ContentType: variant.contentType, SizeBytes: int64(128 - index*16), + SHA256: strings.Repeat(string(rune('a'+index)), 64), Width: 320 - index*32, Height: 200 - index*20, + ObjectKey: fmt.Sprintf("attachments/%s/v%d/%s.%s", attachmentID, pipelineVersion, variant.name, variant.extension), + PipelineVersion: pipelineVersion, + }) + } + return variants +} + func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { t.Helper() var count int diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 267b39d..beb843c 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -11,6 +11,7 @@ import ( type MetadataStore interface { ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) + AttachmentObjectKeys(context.Context, string) ([]string, error) AttachmentExistsByObjectKey(context.Context, string) (bool, error) ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error) DeleteAttachmentIfDeleting(context.Context, string) (bool, error) @@ -23,6 +24,7 @@ type ObjectStore interface { type Config struct { Prefix string + OrphanPrefixes []string PageSize int GraceTime time.Duration UnreferencedRetention time.Duration @@ -65,6 +67,11 @@ func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconcile if config.UnreferencedRetention <= 0 { return nil, errors.New("unreferenced retention must be positive") } + for _, prefix := range config.OrphanPrefixes { + if prefix == "" || prefix == config.Prefix { + return nil, errors.New("orphan prefix must be non-empty and distinct") + } + } if config.Now == nil { config.Now = time.Now } @@ -113,9 +120,35 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) { } } } + for _, prefix := range r.config.OrphanPrefixes { + if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil { + return report, err + } + } return report, nil } +func (r *Reconciler) scanOrphanPrefix( + ctx context.Context, + prefix string, + cutoff time.Time, + report *Report, +) error { + objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize) + if err != nil { + return err + } + for objects.current != nil { + if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil { + return err + } + if err := objects.advance(ctx); err != nil { + return err + } + } + return nil +} + func (r *Reconciler) handlePair( ctx context.Context, metadata model.Attachment, @@ -126,6 +159,9 @@ func (r *Reconciler) handlePair( report.MetadataScanned++ report.ObjectsScanned++ switch metadata.Status { + case model.AttachmentQueued, model.AttachmentProcessing, model.AttachmentFailed: + report.Consistent++ + return nil case model.AttachmentPending: if metadata.CreatedAt.After(anomalyCutoff) { report.FreshPendingAttachments++ @@ -176,10 +212,16 @@ func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Att if !r.config.Apply { return nil } - if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil { - return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err) + objectKeys, err := r.metadata.AttachmentObjectKeys(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("list attachment %q object keys: %w", attachment.ID, err) + } + for _, objectKey := range objectKeys { + if err := r.objects.Delete(ctx, objectKey); err != nil { + return fmt.Errorf("delete attachment object %q: %w", objectKey, err) + } + report.DeletedAttachmentObjects++ } - report.DeletedAttachmentObjects++ deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID) if err != nil { return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err) @@ -197,6 +239,14 @@ func (r *Reconciler) handleOrphanObject( report *Report, ) error { report.ObjectsScanned++ + exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) + if err != nil { + return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) + } + if exists { + report.Consistent++ + return nil + } if object.LastModified.After(cutoff) { report.FreshOrphanObjects++ return nil @@ -205,13 +255,6 @@ func (r *Reconciler) handleOrphanObject( if !r.config.Apply { return nil } - exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) - if err != nil { - return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) - } - if exists { - return nil - } if err := r.objects.Delete(ctx, object.Key); err != nil { return fmt.Errorf("delete orphan object %q: %w", object.Key, err) } @@ -226,6 +269,19 @@ func (r *Reconciler) handleMissingObject( report *Report, ) error { report.MetadataScanned++ + if metadata.Status == model.AttachmentQueued || + metadata.Status == model.AttachmentProcessing || + metadata.Status == model.AttachmentFailed { + return nil + } + if metadata.Status == model.AttachmentPending { + if metadata.CreatedAt.After(cutoff) { + report.FreshPendingAttachments++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, cutoff, report) + } if metadata.Status == model.AttachmentDeleting { return r.deleteClaimedMetadata(ctx, metadata, report) } diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 26c2d92..8e1c1b3 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -21,6 +21,7 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) { "attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false), "attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false), "attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false), + "staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false), }} objects := &objectStub{items: map[string]model.StoredObject{ "attachments/a/original": {Key: "attachments/a/original", LastModified: old}, @@ -28,27 +29,33 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) { "attachments/c/original": {Key: "attachments/c/original", LastModified: old}, "attachments/d/original": {Key: "attachments/d/original", LastModified: fresh}, "attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old}, + "staging/orphan/original": {Key: "staging/orphan/original", LastModified: old}, + "staging/pending/original": {Key: "staging/pending/original", LastModified: old}, }} reconciler, err := New(metadata, objects, Config{ Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour, + OrphanPrefixes: []string{"staging/"}, UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, }) require.NoError(t, err) report, err := reconciler.Run(context.Background()) require.NoError(t, err) - require.Equal(t, 1, report.OldPendingAttachments) + require.Equal(t, 2, report.OldPendingAttachments) require.Equal(t, 1, report.OldUnreferencedAttachments) require.Equal(t, 1, report.FreshUnreferencedAttachments) - require.Equal(t, 1, report.DeletedOrphanObjects) - require.Equal(t, 2, report.DeletedAttachmentObjects) - require.Equal(t, 3, report.DeletedUnreferencedRows) + require.Equal(t, 2, report.DeletedOrphanObjects) + require.Equal(t, 3, report.DeletedAttachmentObjects) + require.Equal(t, 4, report.DeletedUnreferencedRows) require.Contains(t, metadata.items, "attachments/b/original") require.Contains(t, metadata.items, "attachments/d/original") require.NotContains(t, metadata.items, "attachments/a/original") require.NotContains(t, metadata.items, "attachments/c/original") require.NotContains(t, metadata.items, "attachments/missing/original") + require.NotContains(t, metadata.items, "staging/pending/original") require.NotContains(t, objects.items, "attachments/orphan/original") + require.NotContains(t, objects.items, "staging/orphan/original") + require.NotContains(t, objects.items, "staging/pending/original") repeated, err := reconciler.Run(context.Background()) require.NoError(t, err) @@ -75,6 +82,98 @@ func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { require.Contains(t, objects.items, "attachments/a/original") } +func TestReconcilerDoesNotReportTrackedStagingObjectAsOrphan(t *testing.T) { + now := time.Now().UTC() + key := "staging/a/original" + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentQueued, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + key: {Key: key, LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", OrphanPrefixes: []string{"staging/"}, PageSize: 1, + GraceTime: time.Minute, UnreferencedRetention: time.Hour, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.OldOrphanObjects) + require.Zero(t, report.FreshOrphanObjects) + require.Contains(t, objects.items, key) +} + +func TestReconcilerPreservesAttachmentDuringReprocessing(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + for _, status := range []string{ + model.AttachmentQueued, + model.AttachmentProcessing, + model.AttachmentFailed, + } { + t.Run(status, func(t *testing.T) { + original := attachment("a", "attachments/a/v1/original.png", status, now.Add(-48*time.Hour), false) + original.PipelineVersion = 2 + missing := attachment("a", "attachments/a/v1/optimized.webp", status, now.Add(-48*time.Hour), false) + missing.PipelineVersion = 2 + metadata := &metadataStub{items: map[string]model.Attachment{ + original.ObjectKey: original, + missing.ObjectKey: missing, + }} + objects := &objectStub{items: map[string]model.StoredObject{ + original.ObjectKey: {Key: original.ObjectKey, LastModified: now.Add(-48 * time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedAttachmentObjects) + require.Zero(t, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, original.ObjectKey) + require.Contains(t, metadata.items, missing.ObjectKey) + require.Contains(t, objects.items, original.ObjectKey) + }) + } +} + +func TestReconcilerDeletesAllAttachmentVariantsOnce(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + keys := []string{ + "attachments/a/v1/crop.jpg", + "attachments/a/v1/optimized.webp", + "attachments/a/v1/original.png", + "attachments/a/v1/thumbnail.jpg", + } + metadata := &metadataStub{items: make(map[string]model.Attachment, len(keys))} + objects := &objectStub{items: make(map[string]model.StoredObject, len(keys))} + for _, key := range keys { + metadata.items[key] = attachment("a", key, model.AttachmentReady, old, false) + objects.items[key] = model.StoredObject{Key: key, LastModified: old} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, len(keys), report.DeletedAttachmentObjects) + require.Equal(t, 1, report.DeletedUnreferencedRows) + require.Empty(t, metadata.items) + require.Empty(t, objects.items) + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + func TestNewRejectsInvalidRetention(t *testing.T) { _, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1}) require.Error(t, err) @@ -93,6 +192,17 @@ type metadataStub struct { appeared map[string]bool } +func (s *metadataStub) AttachmentObjectKeys(_ context.Context, attachmentID string) ([]string, error) { + var keys []string + for key, item := range s.items { + if item.ID == attachmentID { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys, nil +} + func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) { keys := make([]string, 0, len(s.items)) for key := range s.items { @@ -122,6 +232,7 @@ func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string } func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) { + claim := false for key, item := range s.items { if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced { continue @@ -135,19 +246,20 @@ func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate m } item.Status = model.AttachmentDeleting s.items[key] = item - return true, nil + claim = true } - return false, nil + return claim, nil } func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) { + deleted := false for key, item := range s.items { if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced { delete(s.items, key) - return true, nil + deleted = true } } - return false, nil + return deleted, nil } type objectStub struct { diff --git a/lab5.env.example b/lab5.env.example index 5ad34dc..5d3809a 100644 --- a/lab5.env.example +++ b/lab5.env.example @@ -1,6 +1,7 @@ # Адрес и внутренний порт HTTP-сервера. APP_HOST=0.0.0.0 APP_PORT=8080 +MESSENGER_CONFIG_PATH=config/messenger.yaml # Адреса публикации сервисов на локальной машине. API_BIND_ADDRESS=127.0.0.1:8080 @@ -8,11 +9,6 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081 PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 GRAFANA_BIND_ADDRESS=127.0.0.1:3000 -# Время жизни сессии, идемпотентности и используемое хранилище. -SESSION_TTL_SECONDS=3600 -IDEMPOTENCY_TTL_SECONDS=86400 -SESSION_STORE=valkey - # Число API-реплик за Nginx. API_REPLICAS=2 @@ -26,6 +22,10 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge VALKEY_PASSWORD=local-valkey-password CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 +# Redis Streams используется только как устойчивый брокер заданий. +REDIS_BROKER_PASSWORD=local-redis-broker-password +BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 + # Административные данные MinIO используются только при bootstrap. MINIO_ROOT_USER=local-minio-admin MINIO_ROOT_PASSWORD=local-minio-admin-password @@ -36,36 +36,6 @@ S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY=messenger-app S3_SECRET_KEY=local-messenger-s3-password -S3_USE_TLS=false -S3_TIMEOUT_SECONDS=5 -ATTACHMENT_MAX_BYTES=10485760 -ATTACHMENT_MAX_PIXELS=40000000 -ATTACHMENT_BODY_MEMORY_BYTES=67108864 -ATTACHMENT_DECODE_MEMORY_BYTES=402653184 - -# Размер пула и обслуживание соединений PostgreSQL. -DB_MIN_CONNS=1 -DB_MAX_CONNS=10 -DB_MAX_CONNECTION_AGE_SECONDS=1800 -DB_MAX_CONNECTION_IDLE_SECONDS=300 -DB_HEALTH_CHECK_SECONDS=30 - -# Таймауты зависимостей, миграций и HTTP-запросов. -DEPENDENCY_TIMEOUT_SECONDS=5 -MIGRATION_TIMEOUT_SECONDS=30 -REQUEST_TIMEOUT_SECONDS=5 -HTTP_READ_HEADER_TIMEOUT_SECONDS=5 -HTTP_READ_TIMEOUT_SECONDS=10 -HTTP_WRITE_TIMEOUT_SECONDS=10 -HTTP_IDLE_TIMEOUT_SECONDS=60 -HTTP_SHUTDOWN_TIMEOUT_SECONDS=5 - -# Параметры Argon2id для новых хешей паролей. -PASSWORD_MEMORY_KIB=65536 -PASSWORD_ITERATIONS=3 -PASSWORD_PARALLELISM=2 -PASSWORD_SALT_LENGTH=16 -PASSWORD_KEY_LENGTH=32 # Порт Swagger внутри конфигурации CORS API. SWAGGER_PORT=8081 @@ -91,6 +61,12 @@ POSTGRES_CPUS=2 POSTGRES_MEMORY=1g VALKEY_CPUS=1 VALKEY_MEMORY=256m +REDIS_BROKER_CPUS=1 +REDIS_BROKER_MEMORY=256m +DISPATCHER_CPUS=0.5 +DISPATCHER_MEMORY=128m +WORKER_CPUS=2 +WORKER_MEMORY=1g S3_CPUS=1 S3_MEMORY=512m S3_BOOTSTRAP_CPUS=0.25 diff --git a/migrations/0006_async_attachment_pipeline.sql b/migrations/0006_async_attachment_pipeline.sql new file mode 100644 index 0000000..2b061b6 --- /dev/null +++ b/migrations/0006_async_attachment_pipeline.sql @@ -0,0 +1,135 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status, + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN error_code TEXT, + ADD COLUMN staging_key TEXT; + +UPDATE attachments +SET staging_key = object_key +WHERE status = 'pending'; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK ( + status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting') + ), + ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1), + ADD CONSTRAINT attachments_error_code CHECK ( + (status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR + (status <> 'failed' AND error_code IS NULL) + ), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'ready' AND ready_at IS NOT NULL) OR + (status <> 'ready') + ), + ADD CONSTRAINT attachments_staging_key_length CHECK ( + staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024 + ); + +CREATE INDEX attachments_status_updated_at_idx + ON attachments (status, updated_at, id); + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued', + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + lease_token UUID, + lease_owner TEXT, + lease_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_lease CHECK ( + (status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR + (status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL) + ), + CONSTRAINT attachment_jobs_error_code CHECK ( + error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (available_at, id) + WHERE status = 'queued'; + +CREATE INDEX attachment_jobs_expired_lease_idx + ON attachment_jobs (lease_until, id) + WHERE status = 'processing'; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key), + CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE TABLE outbox_events ( + id UUID PRIMARY KEY, + topic TEXT NOT NULL, + partition_key TEXT NOT NULL, + payload JSONB NOT NULL, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + publish_tries INTEGER NOT NULL DEFAULT 0, + claim_token UUID, + claim_until TIMESTAMPTZ, + last_error_code TEXT, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt), + CONSTRAINT outbox_events_attempt CHECK (attempt >= 1), + CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0), + CONSTRAINT outbox_events_claim CHECK ( + (claim_token IS NULL AND claim_until IS NULL) OR + (claim_token IS NOT NULL AND claim_until IS NOT NULL) + ), + CONSTRAINT outbox_events_last_error_code CHECK ( + last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX outbox_events_due_idx + ON outbox_events (available_at, id) + WHERE published_at IS NULL; + +CREATE TABLE attachment_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + previous_attempt INTEGER NOT NULL, + new_attempt INTEGER NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1), + CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128), + CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512) +); + +CREATE INDEX attachment_replay_audit_job_created_at_idx + ON attachment_replay_audit (job_id, created_at); diff --git a/migrations/0007_outbox_retention.sql b/migrations/0007_outbox_retention.sql new file mode 100644 index 0000000..cc936f6 --- /dev/null +++ b/migrations/0007_outbox_retention.sql @@ -0,0 +1,3 @@ +CREATE INDEX outbox_events_published_at_idx + ON outbox_events (published_at, id) + WHERE published_at IS NOT NULL; diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json index 0478e84..de0ef2d 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger.json @@ -1,143 +1,918 @@ { - "annotations": {"list": []}, + "annotations": { + "list": [] + }, "editable": false, "graphTooltip": 1, "panels": [ { - "type": "timeseries", "title": "API requests per second", "id": 1, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, - "targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API RPS by replica", "id": 2, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, - "targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API responses by status class", "id": 3, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, - "targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Healthy API replicas", "id": 4, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, - "targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API latency p95 by replica and route", "id": 5, - "gridPos": {"h": 8, "w": 24, "x": 0, "y": 16}, - "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, - "targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "API in flight", "id": 6, - "gridPos": {"h": 8, "w": 8, "x": 0, "y": 24}, - "targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Go heap by replica", "id": 7, - "gridPos": {"h": 8, "w": 8, "x": 8, "y": 24}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Idempotency replays", "id": 8, - "gridPos": {"h": 8, "w": 8, "x": 16, "y": 24}, - "targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Nginx connections and requests", "id": 9, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, - "targets": [ - {"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"}, - {"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"} + "type": "row", + "title": "API and edge", + "id": 100, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [ + { + "type": "timeseries", + "title": "API requests by route", + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API responses by status class", + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{status_class}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API latency p95 by route", + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API replicas and requests in flight", + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "targets": [ + { + "expr": "sum(up{job=\"messenger\"})", + "legendFormat": "healthy replicas", + "refId": "A" + }, + { + "expr": "sum(messenger_http_requests_in_flight)", + "legendFormat": "in flight", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API CPU usage", + "id": 5, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(process_cpu_seconds_total{job=\"messenger\"}[1m]))", + "legendFormat": "API", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Nginx requests and connections", + "id": 6, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(nginx_http_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "A" + }, + { + "expr": "sum(nginx_connections_active)", + "legendFormat": "active connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } ] }, { - "type": "timeseries", "title": "PostgreSQL connections", "id": 10, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, - "targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}] - }, - { - "type": "timeseries", "title": "PostgreSQL transactions", "id": 11, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 40}, - "targets": [ - {"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"}, - {"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"} + "type": "row", + "title": "PostgreSQL", + "id": 101, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "panels": [ + { + "type": "timeseries", + "title": "PostgreSQL availability and connections", + "id": 7, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "targets": [ + { + "expr": "up{job=\"postgres\"}", + "legendFormat": "up", + "refId": "A" + }, + { + "expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", + "legendFormat": "connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL transactions", + "id": 8, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "targets": [ + { + "expr": "sum(rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m]))", + "legendFormat": "commit/s", + "refId": "A" + }, + { + "expr": "sum(rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m]))", + "legendFormat": "rollback/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL database size", + "id": 9, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "targets": [ + { + "expr": "pg_database_size_bytes{datname=\"messenger\"}", + "legendFormat": "messenger", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment jobs by status", + "id": 10, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_count", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Oldest attachment job", + "id": 11, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_oldest_age_seconds", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Pending outbox", + "id": 12, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_outbox_pending", + "legendFormat": "events", + "refId": "A" + }, + { + "expr": "messenger_outbox_oldest_pending_age_seconds", + "legendFormat": "oldest age, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } + } ] }, { - "type": "timeseries", "title": "PostgreSQL database size", "id": 12, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 40}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Valkey memory", "id": 13, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 48}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Valkey keys and commands", "id": 14, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 48}, - "targets": [ - {"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"}, - {"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"} + "type": "row", + "title": "Valkey and Redis", + "id": 102, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "panels": [ + { + "type": "timeseries", + "title": "Valkey and Redis availability", + "id": 13, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "targets": [ + { + "expr": "up{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "up{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis memory", + "id": 14, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "targets": [ + { + "expr": "redis_memory_used_bytes{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "redis_memory_used_bytes{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey cache hits and misses", + "id": 15, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_keyspace_hits_total{job=\"valkey\"}[1m]))", + "legendFormat": "hits/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_keyspace_misses_total{job=\"valkey\"}[1m]))", + "legendFormat": "misses/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis commands", + "id": 16, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_commands_processed_total{job=\"valkey\"}[1m]))", + "legendFormat": "Valkey commands/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_commands_processed_total{job=\"redis-broker\"}[1m]))", + "legendFormat": "Redis commands/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + } ] }, { - "type": "timeseries", "title": "Attachment uploads by result", "id": 15, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 56}, - "targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Attachment upload latency p95", "id": 16, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 56}, - "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, - "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}] - }, - { - "type": "timeseries", "title": "Attachment upload traffic", "id": 17, - "gridPos": {"h": 8, "w": 8, "x": 0, "y": 64}, - "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}, - "targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}] - }, - { - "type": "timeseries", "title": "MinIO objects", "id": 18, - "gridPos": {"h": 8, "w": 8, "x": 8, "y": 64}, - "targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}] - }, - { - "type": "timeseries", "title": "MinIO requests and errors", "id": 19, - "gridPos": {"h": 8, "w": 8, "x": 16, "y": 64}, - "targets": [ - {"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"}, - {"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"} + "type": "row", + "title": "Object storage", + "id": 103, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "panels": [ + { + "type": "timeseries", + "title": "MinIO availability", + "id": 17, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 68 + }, + "targets": [ + { + "expr": "up{job=\"minio\"}", + "legendFormat": "up", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO objects and requests", + "id": 18, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_object_total)", + "legendFormat": "objects", + "refId": "A" + }, + { + "expr": "sum(rate(minio_s3_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "B" + }, + { + "expr": "sum(rate(minio_s3_requests_errors_total[1m]))", + "legendFormat": "errors/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO storage", + "id": 19, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_total_bytes)", + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(minio_cluster_capacity_usable_free_bytes)", + "legendFormat": "free", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + } ] }, { - "type": "timeseries", "title": "MinIO availability", "id": 20, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 72}, - "targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}] - }, - { - "type": "timeseries", "title": "MinIO storage", "id": 21, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 72}, - "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, - "targets": [ - {"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"}, - {"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"} + "type": "row", + "title": "Attachment pipeline", + "id": 104, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 76 + }, + "panels": [ + { + "type": "timeseries", + "title": "Pipeline availability and consumers", + "id": 20, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "targets": [ + { + "expr": "sum(up{job=\"attachment-worker\"})", + "legendFormat": "workers up", + "refId": "A" + }, + { + "expr": "sum(up{job=\"outbox-dispatcher\"})", + "legendFormat": "dispatcher up", + "refId": "B" + }, + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "active consumers", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment upload results", + "id": 21, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", + "legendFormat": "bytes/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + } + }, + { + "type": "timeseries", + "title": "Worker throughput by result", + "id": 22, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_worker_jobs_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Worker processing latency p95", + "id": 23, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_worker_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", + "legendFormat": "upload p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Worker jobs in flight", + "id": 24, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 93 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "jobs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Retries and reclaimed deliveries", + "id": 25, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_attachment_worker_retries_total[1m]))", + "legendFormat": "retries/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_worker_reclaimed_deliveries_total[1m]))", + "legendFormat": "reclaimed/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Outbox dispatcher", + "id": 26, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_outbox_published_total[1m]))", + "legendFormat": "published/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_outbox_publish_failures_total[1m]))", + "legendFormat": "failures/s", + "refId": "B" + }, + { + "expr": "sum(rate(messenger_outbox_recovered_leases_total[1m]))", + "legendFormat": "recovered/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Redis Stream lag, PEL and DLQ", + "id": 27, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_lag{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "lag", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_messages_pending{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "PEL", + "refId": "B" + }, + { + "expr": "max(redis_stream_length{job=\"redis-broker\",stream=\"attachment-jobs-failed\"})", + "legendFormat": "DLQ", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Redis Stream consumers", + "id": 28, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "consumers", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_consumer_idle_seconds{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "max idle, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } + } ] } ], "refresh": "5s", "schemaVersion": 41, - "tags": ["highload", "lab4", "nginx", "failover"], - "templating": {"list": []}, - "time": {"from": "now-15m", "to": "now"}, + "tags": [ + "highload", + "lab5", + "pipeline" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, "timezone": "browser", - "title": "Messenger lab4 overview", - "uid": "messenger-overview", + "title": "Messenger lab5 overview", + "uid": "messenger-lab5", "version": 3 } diff --git a/monitoring/postgres-queries.yml b/monitoring/postgres-queries.yml new file mode 100644 index 0000000..7666281 --- /dev/null +++ b/monitoring/postgres-queries.yml @@ -0,0 +1,39 @@ +messenger_attachment_jobs: + query: | + WITH statuses(status) AS ( + VALUES ('queued'), ('processing'), ('failed') + ) + SELECT + statuses.status, + COUNT(j.id)::bigint AS count, + COALESCE(EXTRACT(EPOCH FROM (clock_timestamp() - MIN(j.created_at))), 0)::double precision AS oldest_age_seconds + FROM statuses + LEFT JOIN attachment_jobs j ON j.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: Attachment job status. + - count: + usage: GAUGE + description: Attachment jobs in this status. + - oldest_age_seconds: + usage: GAUGE + description: Age of the oldest attachment job in this status. + +messenger_outbox: + query: | + SELECT + COUNT(*) FILTER (WHERE published_at IS NULL)::bigint AS pending, + COALESCE( + EXTRACT(EPOCH FROM (clock_timestamp() - MIN(created_at) FILTER (WHERE published_at IS NULL))), + 0 + )::double precision AS oldest_pending_age_seconds + FROM outbox_events + metrics: + - pending: + usage: GAUGE + description: Unpublished outbox events. + - oldest_pending_age_seconds: + usage: GAUGE + description: Age of the oldest unpublished outbox event. diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 83721c4..1869184 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -22,6 +22,21 @@ scrape_configs: static_configs: - targets: ["valkey-exporter:9121"] + - job_name: redis-broker + static_configs: + - targets: ["redis-broker-exporter:9121"] + + - job_name: attachment-worker + dns_sd_configs: + - names: ["worker"] + type: A + port: 9091 + refresh_interval: 2s + + - job_name: outbox-dispatcher + static_configs: + - targets: ["dispatcher:9091"] + - job_name: minio metrics_path: /minio/v2/metrics/cluster static_configs: diff --git a/scripts/restart.sh b/scripts/restart.sh index a724583..d587251 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,8 +2,8 @@ set -eu -docker compose stop nginx api valkey postgres s3 -docker compose start postgres valkey s3 +docker compose stop nginx api worker dispatcher redis-broker valkey postgres s3 +docker compose start postgres valkey redis-broker s3 docker compose run --rm s3-bootstrap docker compose run --rm migrate docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index d02dc02..695e3f8 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -150,7 +150,7 @@ fi cp "$upstream_file" "$upstream_backup" compose pull api nginx swagger prometheus grafana s3 s3-bootstrap -compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3 +compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3 compose run --rm --no-deps s3-bootstrap compose run --rm --no-deps migrate @@ -208,7 +208,7 @@ for container in $candidate_pool; do done smoke_check -compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana +compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana printf '%s\n' "$candidate_pool" > "$active_containers" cp "$manifest" "$current_manifest" diff --git a/scripts/test-lab5-acceptance.sh b/scripts/test-lab5-acceptance.sh new file mode 100755 index 0000000..f62397c --- /dev/null +++ b/scripts/test-lab5-acceptance.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -eu + +state_file=$(mktemp) + +restore_pipeline() { + rm -f "$state_file" + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM + +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file" +docker compose restart api redis-broker >/dev/null +docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null +docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file" + diff --git a/scripts/test-lab5.sh b/scripts/test-lab5.sh new file mode 100755 index 0000000..9bf4511 --- /dev/null +++ b/scripts/test-lab5.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -eu + +restore_pipeline() { + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment +