forked from mtusi2026/mtusi2026lab5
Compare commits
9
Commits
baseline/lab4
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ec6cc29d9 | ||
|
|
c5b804426e | ||
|
|
25c80a0072 | ||
|
|
aa5d6a6e16 | ||
|
|
3835106125 | ||
|
|
487015dd2f | ||
|
|
bc090c0f8b | ||
|
|
42336ce7c2 | ||
|
|
8c2ac690a6 |
@@ -46,7 +46,7 @@ jobs:
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
@@ -80,6 +80,8 @@ jobs:
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 90
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
@@ -93,6 +93,8 @@ jobs:
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
|
||||
- name: Publish verified images
|
||||
id: publish
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@ COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go build -p=1 -o /out/messenger ./cmd/messenger
|
||||
RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment
|
||||
RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment
|
||||
|
||||
FROM build AS test
|
||||
|
||||
@@ -35,7 +35,7 @@ FROM alpine:3.24.2
|
||||
|
||||
RUN addgroup -S app && adduser -S -G app -u 10001 app
|
||||
COPY --from=build --chown=app:app /out/messenger /app/messenger
|
||||
COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment
|
||||
COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment
|
||||
|
||||
USER app
|
||||
WORKDIR /app
|
||||
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,9 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-lab5:
|
||||
./scripts/test-lab5.sh
|
||||
|
||||
test-lab5-acceptance:
|
||||
./scripts/test-lab5-acceptance.sh
|
||||
|
||||
@@ -1,37 +1,182 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: Пластинин Денис, БВТ2303.
|
||||
- Целевая оценка: 5.
|
||||
- Выбранный трек: 2, доказанная идемпотентность versioned pipeline.
|
||||
- Ветка решения: `solution/bvt2303-plastinin`.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Обычные настройки приложения находятся в `config/messenger.yaml`; `.env` содержит адреса окружения, локальные credentials и параметры контейнеров.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cp lab5.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 180
|
||||
make test
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
После запуска доступны:
|
||||
|
||||
- API через Nginx: `http://localhost:8080`;
|
||||
- Swagger UI: `http://localhost:8081`;
|
||||
- Prometheus: `http://localhost:9090`;
|
||||
- Grafana: `http://localhost:3000`, пользователь `admin`, пароль берётся из `GRAFANA_ADMIN_PASSWORD`.
|
||||
|
||||
Интеграционные проверки PostgreSQL и Valkey запускаются отдельно:
|
||||
|
||||
```sh
|
||||
docker compose --profile tests run --rm --build tests
|
||||
```
|
||||
|
||||
Эксперимент трека 5 временно останавливает обычные dispatcher и worker, запускает управляемые сбои, затем возвращает процессы в работу:
|
||||
|
||||
```sh
|
||||
make test-lab5
|
||||
```
|
||||
|
||||
Обязательный сценарий с остановленным worker и разрывом commit → publish:
|
||||
|
||||
```sh
|
||||
make test-lab5-acceptance
|
||||
```
|
||||
|
||||
Обычная остановка сохраняет volumes:
|
||||
|
||||
```sh
|
||||
docker compose down
|
||||
```
|
||||
|
||||
Намеренное удаление всех учебных данных:
|
||||
|
||||
```sh
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
## Архитектура
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client --> Nginx
|
||||
Nginx --> API1[API replica 1]
|
||||
Nginx --> API2[API replica 2]
|
||||
API1 --> PostgreSQL
|
||||
API2 --> PostgreSQL
|
||||
API1 --> Valkey[Valkey sessions]
|
||||
API2 --> Valkey
|
||||
API1 --> S3[(MinIO / S3)]
|
||||
API2 --> S3
|
||||
PostgreSQL -->|outbox lease| Dispatcher
|
||||
Dispatcher -->|XADD| Redis[(Redis Streams + AOF)]
|
||||
Redis -->|XREADGROUP| Worker
|
||||
Worker --> S3
|
||||
Worker -->|lease + result transaction| PostgreSQL
|
||||
Worker -->|XACK after commit| Redis
|
||||
```
|
||||
|
||||
PostgreSQL является источником истины для metadata, состояния job, попытки, lease, версий pipeline, вариантов и outbox. MinIO хранит исходник в staging и готовые объекты. Valkey хранит только сессии. Отдельный Redis используется как durable broker: включены AOF, `appendfsync always`, volume и политика `noeviction`.
|
||||
|
||||
Temporal не используется. В работе один короткий тип workflow без ожидания внешних событий и компенсаций между множеством сервисов. Transactional outbox, job lease и Redis consumer group закрывают требуемые окна отказа с меньшим числом компонентов. Добавление Temporal потребовало бы Temporal Service и отдельного persistent store, при этом PostgreSQL job/outbox всё равно понадобился бы для атомарной связи с attachment metadata.
|
||||
|
||||
## Граница ответа 202
|
||||
|
||||
API полностью читает и проверяет PNG/JPEG: MIME, фактический формат, размер и число пикселей. Затем создаётся `pending` metadata, а байты записываются в приватный ключ `staging/<attachment_id>/<uuid>` в S3. После успешной записи одна транзакция PostgreSQL переводит attachment в `queued`, создаёт job и outbox event. Только после commit клиент получает `202 Accepted` со snapshot состояния `queued`.
|
||||
|
||||
Если запись staging не удалась, `202` не возвращается. Если соединение потеряно после commit, сохранённый outbox остаётся источником восстановления. Бинарные данные, S3 ключи и credentials в Redis не передаются: envelope содержит только schema version, job ID, attachment ID, pipeline version, attempt, время и trace context.
|
||||
|
||||
## Обработка изображений
|
||||
|
||||
Worker повторно проверяет доверенный staging объект и создаёт четыре content-addressed объекта:
|
||||
|
||||
| Вариант | Формат | Правило |
|
||||
| --- | --- | --- |
|
||||
| `original` | исходный PNG/JPEG | байты не изменяются |
|
||||
| `thumbnail` | JPEG quality 85 | fit в `256×256`, без увеличения |
|
||||
| `crop` | JPEG quality 85 | центральный квадрат `128×128`, увеличение разрешено |
|
||||
| `optimized` | WebP quality 82, method 4 | fit в `1280×1280`, без увеличения |
|
||||
|
||||
EXIF orientation применяется к производным вариантам. Прозрачность сводится на белый фон. Повторное кодирование удаляет metadata. Ключ эффекта имеет вид `attachments/<id>/pipelines/<version>/<variant>/<sha256>.<ext>`, поэтому повтор той же операции записывает тот же объект.
|
||||
|
||||
Concurrency и prefetch задаются через `worker.concurrency` и `worker.prefetch`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `worker.memory_bytes` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд. Worker продлевает lease каждые 5 секунд; потеря lease отменяет обработку. Decode и собственные циклы преобразования проверяют context во время работы.
|
||||
|
||||
## Доставка, retry и восстановление
|
||||
|
||||
Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль. Опубликованные события хранятся семь суток и удаляются раз в минуту пакетами до 1000 строк, не задерживая первый dispatch после запуска.
|
||||
|
||||
Worker читает consumer group через `XREADGROUP`. При старте обработки PostgreSQL атомарно проверяет `job_id`, `pipeline_version`, `attempt`, выдаёт lease token и переводит attachment в `processing`. Результат применяется только при совпадении всех fencing полей и живом lease. `XACK` выполняется после commit результата. После ACK обработанная запись удаляется из основного stream; pending записи не вытесняются. Failed stream ограничен приблизительно 10 000 записями.
|
||||
|
||||
Временная ошибка переводит job обратно в `queued`, увеличивает `attempt` и в той же транзакции создаёт новый outbox event. Задержка растёт экспоненциально и получает детерминированный jitter до 25%. Постоянные ошибки (`invalid_image`, `image_too_large`, `source_missing`) и исчерпание пяти попыток переводят attachment в `failed`. Явный replay выполняется командой:
|
||||
|
||||
```sh
|
||||
docker compose run --rm api /app/messenger replay-attachment <attachment-id> <actor> <reason>
|
||||
```
|
||||
|
||||
Replay допустим только для `failed`, увеличивает attempt и сохраняет audit record. Повтор `ready` job становится no-op. `XAUTOCLAIM` возвращает зависшие Redis deliveries, а PostgreSQL recovery возвращает job с истёкшим lease в очередь. Старый consumer не может подтвердить delivery после передачи новому consumer: Lua script проверяет владельца PEL перед `XACK` или переносом в failed stream.
|
||||
|
||||
## Эксперимент трека 5
|
||||
|
||||
### Гипотеза
|
||||
|
||||
At-least-once доставка не создаст второй логический результат, если effect адресуется по `(attachment_id, pipeline_version, variant)`, а публикация metadata защищена `attempt` и lease token. Старая версия pipeline не сможет перезаписать текущую.
|
||||
|
||||
### Методика
|
||||
|
||||
Go-программа `cmd/lab5experiment` работает с настоящими PostgreSQL, Redis Streams и MinIO из Compose. Используется PNG `320×200`. Обычные worker и dispatcher на время прогона остановлены, чтобы порядок событий задавался экспериментом.
|
||||
|
||||
Автоматически выполняются:
|
||||
|
||||
1. crash после claim, но до side effect;
|
||||
2. crash после записи всех S3 объектов, но до commit результата;
|
||||
3. crash после commit результата, но до consumer ACK;
|
||||
4. две дополнительные доставки уже завершённой job;
|
||||
5. постоянная ошибка повреждённого staging и явный replay;
|
||||
6. завершение pipeline v2, после которого доставляется устаревшая job v1.
|
||||
|
||||
Для первых двух сценариев ожидается истечение lease, увеличение attempt и завершение следующей попыткой. В третьем и четвёртом сценариях повтор должен подтвердиться как no-op. Повреждённый staging дважды завершается одинаковым безопасным `invalid_image`, причём replay увеличивает attempt и проходит только через audit/outbox. После готовой версии v2 поздняя v1 не меняет metadata.
|
||||
|
||||
### Результат
|
||||
|
||||
Raw summaries сохранены в `evidence/lab5-idempotency.json` и `evidence/lab5-resilience.json`.
|
||||
|
||||
| Воздействие | До восстановления | После восстановления |
|
||||
| --- | --- | --- |
|
||||
| crash до side effect | 0 конечных объектов | attempt 2, `ready`, 4 объекта |
|
||||
| crash после side effect | 4 объекта, DB ещё `processing` | attempt 2, `ready`, те же 4 объекта |
|
||||
| crash после commit до ACK | DB уже `ready`, 4 объекта | attempt 1, повтор no-op, 4 объекта |
|
||||
| две duplicate deliveries | 4 объекта | 4 объекта |
|
||||
| permanent failure и replay | `failed`, `invalid_image` | replay attempt 2, снова `failed`, `invalid_image` |
|
||||
| stale v1 после готовой v2 | `ready` v2 | остаётся `ready` v2, 8 объектов для двух версий |
|
||||
|
||||
Все пять подтверждённых logical jobs дошли до терминального состояния: четыре `ready`, одна ожидаемо `failed`. Инварианты `accepted_jobs_reach_terminal_state`, `duplicate_effect_is_unique` и `old_pipeline_cannot_replace_new` получили `true`. Прогон занял 6981 ms.
|
||||
|
||||
Это не exactly-once доставка: Redis может доставить одно сообщение несколько раз, а dispatcher может повторить `XADD`. Exactly-once относится только к наблюдаемому эффекту, который достигается идемпотентными ключами и compare-and-set в PostgreSQL.
|
||||
|
||||
## Мониторинг и CI
|
||||
|
||||
Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard разделён на блоки API, PostgreSQL, Valkey/Redis, object storage и attachment pipeline. В нём показаны timing основных ручек, CPU, соединения и транзакции PostgreSQL, cache hit/miss, память, доступность worker/dispatcher, throughput, p95 обработки, in-flight jobs, retry/reclaim, outbox backlog, Redis Streams lag, PEL, DLQ и idle consumers.
|
||||
|
||||
CI для pull request и `main` выполняет contract check, OpenAPI validation, formatting, lint, race tests, build, публичный smoke, интеграционные тесты, persistence/failover lab4 и fault injection lab5. Production workflow запускается вручную и требует approval environment перед rolling deployment.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Команда | Результат |
|
||||
| --- | --- | --- |
|
||||
| Публичный API lab5 | `make test` | PASS: queued → ready, original и thumbnail проверены |
|
||||
| PostgreSQL и Valkey | `docker compose --profile tests run --rm --build tests` | оба пакета PASS |
|
||||
| Transform | `go test ./internal/messenger/attachment/transform` | thumbnail/crop/WebP, EXIF, alpha и лимиты PASS |
|
||||
| Redis и outbox | `go test ./internal/broker/redisstream ./internal/outbox` | envelope, poison, owner fencing и lost lease PASS |
|
||||
| Worker остановлен, outbox gap, restart broker/API | `make test-lab5-acceptance` | queued link и 409 до worker; после restart `ready`, original сохранён |
|
||||
| Трек 5 | `make test-lab5` | 5/5 jobs terminal, 4 ready, 1 expected failed, все три инварианта true |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Ограничения
|
||||
|
||||
## Эксперименты и отказы
|
||||
Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Decode и собственные циклы обработки прерываются по deadline; библиотечный вызов WebP обрабатывает ограниченный вариант до `1280×1280` и не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
## Источники и библиотеки
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Redis Streams и consumer groups: <https://redis.io/docs/latest/develop/data-types/streams/>.
|
||||
- Redis persistence: <https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/>.
|
||||
- PostgreSQL `SKIP LOCKED`: <https://www.postgresql.org/docs/current/sql-select.html>.
|
||||
- MinIO Go SDK: <https://github.com/minio/minio-go>.
|
||||
- OpenTelemetry trace context: <https://www.w3.org/TR/trace-context/>.
|
||||
- Go-пакеты: `pgx/v5`, `go-redis/v9`, `minio-go/v7`, `deepteams/webp`, Prometheus client.
|
||||
|
||||
@@ -1,676 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
|
||||
)
|
||||
|
||||
const experimentPassword = "Lab4-experiment-password"
|
||||
|
||||
type apiClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
token string
|
||||
}
|
||||
|
||||
type attachment struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type setup struct {
|
||||
client *apiClient
|
||||
chatID string
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(arguments []string, output io.Writer) error {
|
||||
if len(arguments) == 0 {
|
||||
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||
}
|
||||
switch arguments[0] {
|
||||
case "memory":
|
||||
return runMemory(arguments[1:], output)
|
||||
case "consistency":
|
||||
return runConsistency(arguments[1:], output)
|
||||
default:
|
||||
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||
}
|
||||
}
|
||||
|
||||
type memoryResult struct {
|
||||
Replicas int `json:"replicas"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Rounds int `json:"rounds"`
|
||||
SettleTimeMillis int64 `json:"settle_time_ms"`
|
||||
ImageBytes int `json:"image_bytes"`
|
||||
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
|
||||
SuccessfulUploads int64 `json:"successful_uploads"`
|
||||
FailedUploads int64 `json:"failed_uploads"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
|
||||
HeapPeakBytes int64 `json:"heap_peak_bytes"`
|
||||
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
|
||||
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
|
||||
RSSPeakBytes int64 `json:"rss_peak_bytes"`
|
||||
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
|
||||
}
|
||||
|
||||
func runMemory(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
|
||||
replicas := flags.Int("replicas", 2, "expected API replicas")
|
||||
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
|
||||
rounds := flags.Int("rounds", 4, "number of upload waves")
|
||||
width := flags.Int("width", 2048, "image width")
|
||||
height := flags.Int("height", 2048, "image height")
|
||||
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
|
||||
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("invalid memory experiment flags")
|
||||
}
|
||||
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
|
||||
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
fixture, err := jpegFixture(*width, *height)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
|
||||
return err
|
||||
}
|
||||
settle := time.NewTimer(*settleTime)
|
||||
defer settle.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-settle.C:
|
||||
}
|
||||
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
environment, err := prepare(ctx, *baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var successful atomic.Int64
|
||||
var failed atomic.Int64
|
||||
var firstError struct {
|
||||
sync.Mutex
|
||||
err error
|
||||
}
|
||||
heapPeak := heapBaseline
|
||||
rssPeak := rssBaseline
|
||||
loadDone := make(chan struct{})
|
||||
sampleDone := make(chan error, 1)
|
||||
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
|
||||
started := time.Now()
|
||||
for round := range *rounds {
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(*concurrency)
|
||||
for index := range *concurrency {
|
||||
go func(round, index int) {
|
||||
defer workers.Done()
|
||||
reader := newRateReader(fixture, *uploadRate)
|
||||
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
|
||||
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
|
||||
failed.Add(1)
|
||||
firstError.Lock()
|
||||
if firstError.err == nil {
|
||||
firstError.err = err
|
||||
}
|
||||
firstError.Unlock()
|
||||
return
|
||||
}
|
||||
successful.Add(1)
|
||||
}(round, index)
|
||||
}
|
||||
workers.Wait()
|
||||
}
|
||||
close(loadDone)
|
||||
if err := <-sampleDone; err != nil {
|
||||
return err
|
||||
}
|
||||
result := memoryResult{
|
||||
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
|
||||
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
|
||||
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
|
||||
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
|
||||
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
|
||||
}
|
||||
if result.FailedUploads > 0 {
|
||||
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
|
||||
}
|
||||
return writeJSON(output, result)
|
||||
}
|
||||
|
||||
func sampleMemory(
|
||||
ctx context.Context,
|
||||
prometheusURL string,
|
||||
expectedReplicas int,
|
||||
done <-chan struct{},
|
||||
heapPeak *int64,
|
||||
rssPeak *int64,
|
||||
result chan<- error,
|
||||
) {
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
result <- ctx.Err()
|
||||
return
|
||||
case <-done:
|
||||
result <- nil
|
||||
return
|
||||
case <-ticker.C:
|
||||
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
if replicas != int64(expectedReplicas) {
|
||||
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
|
||||
return
|
||||
}
|
||||
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
*heapPeak = max(*heapPeak, heap)
|
||||
*rssPeak = max(*rssPeak, rss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type consistencyResult struct {
|
||||
PageSize int `json:"page_size"`
|
||||
SeededConsistent int `json:"seeded_consistent"`
|
||||
FreshApply reconciliationRun `json:"fresh_apply"`
|
||||
DryRun reconciliationRun `json:"dry_run"`
|
||||
Apply reconciliationRun `json:"apply"`
|
||||
RepeatedApply reconciliationRun `json:"repeated_apply"`
|
||||
FreshPreserved bool `json:"fresh_preserved"`
|
||||
OrphanDeleted bool `json:"orphan_deleted"`
|
||||
UnusedDeleted bool `json:"unused_metadata_deleted"`
|
||||
InUsePreserved bool `json:"in_use_metadata_preserved"`
|
||||
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
|
||||
TotalDurationMillis int64 `json:"total_duration_ms"`
|
||||
}
|
||||
|
||||
type reconciliationRun struct {
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
Report reconcile.Report `json:"report"`
|
||||
}
|
||||
|
||||
func runConsistency(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
pageSize := flags.Int("page-size", 10, "reconciliation page size")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("invalid consistency experiment flags")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
settings, err := config.LoadReconciliation()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metadata := postgres.New(pool)
|
||||
clean, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
initial, err := clean.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if hasAnomalies(initial) {
|
||||
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
environment, err := prepare(ctx, *baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := jpegFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seededConsistent := *pageSize + 1
|
||||
for index := range seededConsistent {
|
||||
filename := fmt.Sprintf("consistent-%03d.jpg", index)
|
||||
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
|
||||
}
|
||||
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
|
||||
return err
|
||||
}
|
||||
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
|
||||
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
|
||||
UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
freshRun, err := measuredReconciliation(ctx, fresh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
freshPreserved := objectExists(ctx, objects, orphanKey) &&
|
||||
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
|
||||
attachmentExists(ctx, pool, abandoned.ID)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 hours'
|
||||
WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("age abandoned attachment: %w", err)
|
||||
}
|
||||
|
||||
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dryRunResult, err := measuredReconciliation(ctx, dryRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apply, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyResult, err := measuredReconciliation(ctx, apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeatedResult, err := measuredReconciliation(ctx, apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result := consistencyResult{
|
||||
PageSize: *pageSize, SeededConsistent: seededConsistent,
|
||||
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
||||
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
||||
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
||||
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||
TotalDurationMillis: time.Since(started).Milliseconds(),
|
||||
}
|
||||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
||||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
|
||||
return fmt.Errorf("unexpected reconciliation result: %+v", result)
|
||||
}
|
||||
return writeJSON(output, result)
|
||||
}
|
||||
|
||||
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
|
||||
started := time.Now()
|
||||
report, err := reconciler.Run(ctx)
|
||||
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
|
||||
}
|
||||
|
||||
func hasAnomalies(report reconcile.Report) bool {
|
||||
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
||||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
||||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
||||
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
|
||||
}
|
||||
|
||||
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
||||
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query attachment keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
keys := make(map[string]string, len(ids))
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var key string
|
||||
if err := rows.Scan(&id, &key); err != nil {
|
||||
return nil, fmt.Errorf("scan attachment key: %w", err)
|
||||
}
|
||||
keys[id] = key
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate attachment keys: %w", err)
|
||||
}
|
||||
if len(keys) != len(ids) {
|
||||
return nil, errors.New("not all attachment keys were found")
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
|
||||
var exists bool
|
||||
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
|
||||
}
|
||||
|
||||
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
|
||||
_, err := objects.Stat(ctx, key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func prepare(ctx context.Context, baseURL string) (setup, error) {
|
||||
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
|
||||
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
|
||||
var user struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
|
||||
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
|
||||
}, &user, ""); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
|
||||
var session struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
client.token = session.Token
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
|
||||
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
|
||||
}, &chat, "Bearer "+client.token); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
return setup{client: client, chatID: chat.ID}, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) upload(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
filename string,
|
||||
body io.Reader,
|
||||
contentLength int64,
|
||||
) (attachment, error) {
|
||||
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
request.ContentLength = contentLength
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "image/jpeg")
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
|
||||
}
|
||||
var result attachment
|
||||
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
|
||||
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
|
||||
"text": "", "attachment_ids": []string{attachmentID},
|
||||
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
|
||||
}
|
||||
|
||||
func (c *apiClient) jsonRequest(
|
||||
ctx context.Context,
|
||||
method string,
|
||||
path string,
|
||||
payload any,
|
||||
result any,
|
||||
authorization string,
|
||||
extraHeader ...string,
|
||||
) error {
|
||||
var body io.Reader
|
||||
if payload != nil {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(encoded)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if payload != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if authorization != "" {
|
||||
request.Header.Set("Authorization", authorization)
|
||||
}
|
||||
for index := 0; index+1 < len(extraHeader); index += 2 {
|
||||
request.Header.Set(extraHeader[index], extraHeader[index+1])
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
|
||||
}
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return json.NewDecoder(response.Body).Decode(result)
|
||||
}
|
||||
|
||||
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||
if err == nil && count == int64(expected) {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func activeReplicasQuery() string {
|
||||
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
|
||||
}
|
||||
|
||||
func freshMetricQuery(metric string) string {
|
||||
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
|
||||
}
|
||||
|
||||
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
values := request.URL.Query()
|
||||
values.Set("query", query)
|
||||
request.URL.RawQuery = values.Encode()
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Value []json.RawMessage `json:"value"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
|
||||
return 0, errors.New("prometheus query returned no scalar value")
|
||||
}
|
||||
var raw string
|
||||
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
return int64(value), err
|
||||
}
|
||||
|
||||
func jpegFixture(width, height int) ([]byte, error) {
|
||||
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
state := uint64(0x9e3779b97f4a7c15)
|
||||
for index := range pixels.Pix {
|
||||
state ^= state << 13
|
||||
state ^= state >> 7
|
||||
state ^= state << 17
|
||||
pixels.Pix[index] = byte(state >> 24)
|
||||
}
|
||||
var encoded bytes.Buffer
|
||||
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encoded.Bytes(), nil
|
||||
}
|
||||
|
||||
type rateReader struct {
|
||||
reader *bytes.Reader
|
||||
rate int64
|
||||
started time.Time
|
||||
read int64
|
||||
}
|
||||
|
||||
func newRateReader(data []byte, rate int64) *rateReader {
|
||||
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
|
||||
}
|
||||
|
||||
func (r *rateReader) Read(buffer []byte) (int, error) {
|
||||
for {
|
||||
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
|
||||
if allowed > 0 {
|
||||
if int64(len(buffer)) > allowed {
|
||||
buffer = buffer[:allowed]
|
||||
}
|
||||
count, err := r.reader.Read(buffer)
|
||||
r.read += int64(count)
|
||||
return count, err
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(output io.Writer, value any) error {
|
||||
encoder := json.NewEncoder(output)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
type apiState struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Token string `json:"token"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
}
|
||||
|
||||
type apiUser struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type apiSession struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
type apiChat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
func runAPIProduce(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("api-produce", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("usage: lab5experiment api-produce [-base-url URL]")
|
||||
}
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
|
||||
password := "Lab5-" + uuid.NewString()
|
||||
owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token, err := createAPISession(client, *baseURL, "qa_"+suffix, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := pngFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var attachment model.Attachment
|
||||
response, err := apiRequest(
|
||||
client, http.MethodPost,
|
||||
*baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png",
|
||||
token, "image/png", fixture, &attachment,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued {
|
||||
return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status)
|
||||
}
|
||||
messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err = apiRequestWithHeaders(
|
||||
client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages",
|
||||
token, "application/json", messageBody, nil,
|
||||
map[string]string{"Idempotency-Key": uuid.NewString()},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("link queued attachment returned %d", response.StatusCode)
|
||||
}
|
||||
response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusConflict {
|
||||
return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode)
|
||||
}
|
||||
return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID})
|
||||
}
|
||||
|
||||
func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("api-verify", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]")
|
||||
}
|
||||
var state apiState
|
||||
if err := json.NewDecoder(input).Decode(&state); err != nil {
|
||||
return fmt.Errorf("decode API scenario state: %w", err)
|
||||
}
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout)
|
||||
defer cancel()
|
||||
var attachment model.Attachment
|
||||
for {
|
||||
response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("attachment metadata returned %d", response.StatusCode)
|
||||
}
|
||||
if attachment.Status == model.AttachmentReady {
|
||||
break
|
||||
}
|
||||
if attachment.Status == model.AttachmentFailed {
|
||||
return fmt.Errorf("attachment failed with %v", attachment.ErrorCode)
|
||||
}
|
||||
if err := wait(ctx, 250*time.Millisecond); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(attachment.Variants) != 3 {
|
||||
return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants))
|
||||
}
|
||||
response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := pngFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) {
|
||||
return errors.New("original changed after worker and broker restart")
|
||||
}
|
||||
response, thumbnail, err := apiBytes(
|
||||
client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" {
|
||||
return errors.New("thumbnail response is invalid")
|
||||
}
|
||||
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail))
|
||||
if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 {
|
||||
return errors.New("thumbnail dimensions are invalid")
|
||||
}
|
||||
return json.NewEncoder(output).Encode(map[string]any{
|
||||
"accepted_while_worker_stopped": true,
|
||||
"accepted_before_outbox_publish": true,
|
||||
"linked_while_queued": true,
|
||||
"content_conflict_while_queued": true,
|
||||
"ready_after_api_broker_restart": true,
|
||||
"original_preserved": true,
|
||||
"thumbnail_dimensions": "256x160",
|
||||
})
|
||||
}
|
||||
|
||||
func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) {
|
||||
body, err := json.Marshal(map[string]string{
|
||||
"username": username, "display_name": username, "password": password,
|
||||
})
|
||||
if err != nil {
|
||||
return apiUser{}, err
|
||||
}
|
||||
var user apiUser
|
||||
response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user)
|
||||
if err != nil {
|
||||
return apiUser{}, err
|
||||
}
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode)
|
||||
}
|
||||
return user, nil
|
||||
}
|
||||
|
||||
func createAPISession(client *http.Client, baseURL, username, password string) (string, error) {
|
||||
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
|
||||
request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("Authorization", "Basic "+credentials)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("create session returned %d", response.StatusCode)
|
||||
}
|
||||
var session apiSession
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return session.Token, nil
|
||||
}
|
||||
|
||||
func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) {
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID},
|
||||
})
|
||||
if err != nil {
|
||||
return apiChat{}, err
|
||||
}
|
||||
var chat apiChat
|
||||
response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat)
|
||||
if err != nil {
|
||||
return apiChat{}, err
|
||||
}
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode)
|
||||
}
|
||||
return chat, nil
|
||||
}
|
||||
|
||||
func apiRequest(
|
||||
client *http.Client,
|
||||
method string,
|
||||
requestURL string,
|
||||
token string,
|
||||
contentType string,
|
||||
body []byte,
|
||||
result any,
|
||||
) (*http.Response, error) {
|
||||
return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil)
|
||||
}
|
||||
|
||||
func apiRequestWithHeaders(
|
||||
client *http.Client,
|
||||
method string,
|
||||
requestURL string,
|
||||
token string,
|
||||
contentType string,
|
||||
body []byte,
|
||||
result any,
|
||||
headers map[string]string,
|
||||
) (*http.Response, error) {
|
||||
request, err := http.NewRequest(method, requestURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if token != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
if contentType != "" {
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
for name, value := range headers {
|
||||
request.Header.Set(name, value)
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if result != nil && response.StatusCode < http.StatusBadRequest {
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) {
|
||||
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
|
||||
return response, body, err
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
)
|
||||
|
||||
const (
|
||||
experimentLease = 2 * time.Second
|
||||
maxAttempts = 5
|
||||
)
|
||||
|
||||
type result struct {
|
||||
StartedAt string `json:"started_at"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
FaultScenarios []scenario `json:"fault_scenarios"`
|
||||
DuplicateDeliveries int `json:"duplicate_deliveries"`
|
||||
V1ObjectCount int `json:"v1_object_count"`
|
||||
V2ObjectCount int `json:"v2_object_count"`
|
||||
AcceptedJobs int `json:"accepted_jobs"`
|
||||
TerminalJobs int `json:"terminal_jobs"`
|
||||
ReadyJobs int `json:"ready_jobs"`
|
||||
FailedJobs int `json:"failed_jobs"`
|
||||
PermanentErrorCode string `json:"permanent_error_code"`
|
||||
ReplayAttempt int `json:"replay_attempt"`
|
||||
InvariantResults map[string]bool `json:"invariant_results"`
|
||||
}
|
||||
|
||||
type scenario struct {
|
||||
Name string `json:"name"`
|
||||
AttemptAfterRecovery int `json:"attempt_after_recovery"`
|
||||
ObjectsBeforeRecovery int `json:"objects_before_recovery"`
|
||||
ObjectsAfterRecovery int `json:"objects_after_recovery"`
|
||||
FinalStatus string `json:"final_status"`
|
||||
}
|
||||
|
||||
type experiment struct {
|
||||
settings config.WorkerConfig
|
||||
repository *postgres.Store
|
||||
objects *objectstore.Store
|
||||
broker *redisstream.Broker
|
||||
dispatcher *outbox.Dispatcher
|
||||
worker *worker.Worker
|
||||
ownerID string
|
||||
chatID string
|
||||
fixture []byte
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(arguments []string, output io.Writer) error {
|
||||
if len(arguments) > 0 {
|
||||
switch arguments[0] {
|
||||
case "api-produce":
|
||||
return runAPIProduce(arguments[1:], output)
|
||||
case "api-verify":
|
||||
return runAPIVerify(arguments[1:], os.Stdin, output)
|
||||
}
|
||||
}
|
||||
flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 {
|
||||
return errors.New("usage: lab5experiment [-timeout 2m]")
|
||||
}
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
||||
defer cancel()
|
||||
experiment, closeExperiment, err := newExperiment(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeExperiment()
|
||||
|
||||
report, err := experiment.run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.StartedAt = started.UTC().Format(time.RFC3339)
|
||||
report.DurationMillis = time.Since(started).Milliseconds()
|
||||
encoder := json.NewEncoder(output)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(report)
|
||||
}
|
||||
|
||||
func newExperiment(ctx context.Context) (*experiment, func(), error) {
|
||||
settings, err := config.LoadWorker()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
redisOptions, err := redis.ParseURL(settings.BrokerURL)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
client := redis.NewClient(redisOptions)
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
consumer := "lab5experiment-" + uuid.NewString()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
||||
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second,
|
||||
ClaimMinIdle: experimentLease, ReadCount: 32,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := broker.Ensure(ctx); err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
repository := postgres.New(pool)
|
||||
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
|
||||
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
|
||||
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
|
||||
Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
processorWorker, err := worker.New(
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
|
||||
worker.Config{
|
||||
WorkerID: consumer, Concurrency: 1,
|
||||
LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond,
|
||||
ProcessingTimeout: time.Second,
|
||||
ReclaimInterval: experimentLease,
|
||||
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
MemoryBytes: settings.MemoryBytes,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
fixture, err := pngFixture(320, 200)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
value := &experiment{
|
||||
settings: settings, repository: repository, objects: objects, broker: broker,
|
||||
dispatcher: dispatcher, worker: processorWorker, fixture: fixture,
|
||||
}
|
||||
if err := value.seed(ctx); err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
closeExperiment := func() {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
}
|
||||
return value, closeExperiment, nil
|
||||
}
|
||||
|
||||
func (e *experiment) seed(ctx context.Context) error {
|
||||
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
|
||||
owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.ownerID = owner.ID
|
||||
e.chatID = chat.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *experiment) run(ctx context.Context) (result, error) {
|
||||
report := result{
|
||||
PipelineVersion: 1,
|
||||
InvariantResults: map[string]bool{
|
||||
"accepted_jobs_reach_terminal_state": false,
|
||||
"duplicate_effect_is_unique": false,
|
||||
"old_pipeline_cannot_replace_new": false,
|
||||
},
|
||||
}
|
||||
for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} {
|
||||
observed, err := e.runLeaseRecovery(ctx, fault)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
report.FaultScenarios = append(report.FaultScenarios, observed)
|
||||
report.AcceptedJobs++
|
||||
if observed.FinalStatus == model.AttachmentReady {
|
||||
report.ReadyJobs++
|
||||
report.TerminalJobs++
|
||||
}
|
||||
}
|
||||
permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
report.AcceptedJobs++
|
||||
report.TerminalJobs++
|
||||
report.FailedJobs++
|
||||
report.PermanentErrorCode = permanentCode
|
||||
report.ReplayAttempt = replayAttempt
|
||||
|
||||
commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
report.FaultScenarios = append(report.FaultScenarios, commitScenario)
|
||||
report.AcceptedJobs++
|
||||
if commitScenario.FinalStatus == model.AttachmentReady {
|
||||
report.ReadyJobs++
|
||||
report.TerminalJobs++
|
||||
}
|
||||
|
||||
payload, err := redisstream.EncodeEnvelope(envelope)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
for range 2 {
|
||||
if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
}
|
||||
duplicateDeliveries, err := e.readDeliveries(ctx, 2)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
for _, delivery := range duplicateDeliveries {
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
}
|
||||
report.DuplicateDeliveries = len(duplicateDeliveries)
|
||||
report.V1ObjectCount, err = e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if report.V1ObjectCount != 4 {
|
||||
return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount)
|
||||
}
|
||||
report.InvariantResults["duplicate_effect_is_unique"] = true
|
||||
|
||||
oldPayload := payload
|
||||
v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1}
|
||||
if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
v2Delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
staleDelivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 {
|
||||
return result{}, errors.New("new pipeline version was not preserved after stale delivery")
|
||||
}
|
||||
report.V2ObjectCount, err = e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if report.V2ObjectCount != 8 {
|
||||
return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount)
|
||||
}
|
||||
report.InvariantResults["old_pipeline_cannot_replace_new"] = true
|
||||
report.AcceptedJobs++
|
||||
report.ReadyJobs++
|
||||
report.TerminalJobs++
|
||||
report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) {
|
||||
attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png")
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil {
|
||||
return "", 0, errors.New("corrupted staging object did not reach failed")
|
||||
}
|
||||
firstCode := *metadata.ErrorCode
|
||||
replayed, err := e.repository.ReplayFailedAttachment(
|
||||
ctx, attachmentID, "lab5experiment", "verify explicit replay fencing",
|
||||
)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
replayDelivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode {
|
||||
return "", 0, errors.New("replayed permanent failure changed terminal classification")
|
||||
}
|
||||
return firstCode, replayed.Attempt, nil
|
||||
}
|
||||
|
||||
func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) {
|
||||
attachmentID, _, err := e.queueAttachment(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) {
|
||||
return scenario{}, fmt.Errorf("fault %s returned %v", fault, err)
|
||||
}
|
||||
before, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if fault == worker.FaultBeforeSideEffect && before != 0 {
|
||||
return scenario{}, errors.New("objects were written before the injected side effect")
|
||||
}
|
||||
if fault == worker.FaultAfterSideEffect && before != 4 {
|
||||
return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before)
|
||||
}
|
||||
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now())
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if recovered.Requeued != 1 {
|
||||
return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued)
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
retry, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if err := e.ackReclaimed(ctx); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
after, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String())
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
return scenario{
|
||||
Name: string(fault), AttemptAfterRecovery: job.Attempt,
|
||||
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) {
|
||||
attachmentID, _, err := e.queueAttachment(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) {
|
||||
return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err)
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
before, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
if err := e.ackReclaimed(ctx); err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
after, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
return scenario{
|
||||
Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt,
|
||||
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
|
||||
}, delivery.Envelope, attachmentID, nil
|
||||
}
|
||||
|
||||
func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) {
|
||||
attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png")
|
||||
return attachmentID, job, err
|
||||
}
|
||||
|
||||
func (e *experiment) queueAttachmentData(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
contentType string,
|
||||
) (string, string, model.AttachmentJob, error) {
|
||||
attachmentID := uuid.NewString()
|
||||
stagingKey := "staging/" + attachmentID + "/experiment"
|
||||
digest := sha256.Sum256(data)
|
||||
_, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{
|
||||
ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png",
|
||||
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
||||
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1}
|
||||
if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
return attachmentID, stagingKey, job, nil
|
||||
}
|
||||
|
||||
func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) {
|
||||
deliveries, err := e.readDeliveries(ctx, 1)
|
||||
if err != nil {
|
||||
return redisstream.Delivery{}, err
|
||||
}
|
||||
return deliveries[0], nil
|
||||
}
|
||||
|
||||
func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) {
|
||||
var result []redisstream.Delivery
|
||||
for len(result) < expected {
|
||||
deliveries, err := e.broker.Read(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, deliveries...)
|
||||
}
|
||||
if len(result) != expected {
|
||||
return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (e *experiment) ackReclaimed(ctx context.Context) error {
|
||||
deliveries, _, err := e.broker.Reclaim(ctx, "0-0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(deliveries) != 1 {
|
||||
return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries))
|
||||
}
|
||||
return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone)
|
||||
}
|
||||
|
||||
func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) {
|
||||
objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if next != "" {
|
||||
return 0, errors.New("unexpected object pagination")
|
||||
}
|
||||
return len(objects), nil
|
||||
}
|
||||
|
||||
func pngFixture(width, height int) ([]byte, error) {
|
||||
value := image.NewNRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
|
||||
}
|
||||
}
|
||||
var output bytes.Buffer
|
||||
if err := png.Encode(&output, value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return output.Bytes(), nil
|
||||
}
|
||||
|
||||
func wait(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
+10
-1
@@ -24,8 +24,17 @@ func run(logger *slog.Logger, arguments []string) error {
|
||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||
return bootstrap.RunMigrations()
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "dispatch" {
|
||||
return bootstrap.RunDispatcher(logger)
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "worker" {
|
||||
return bootstrap.RunWorker(logger)
|
||||
}
|
||||
if arguments[0] == "reconcile-attachments" {
|
||||
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
|
||||
}
|
||||
return errors.New("usage: messenger [migrate|reconcile-attachments]")
|
||||
if arguments[0] == "replay-attachment" {
|
||||
return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout)
|
||||
}
|
||||
return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]")
|
||||
}
|
||||
|
||||
+146
-33
@@ -1,3 +1,9 @@
|
||||
x-messenger-config-volume: &messenger-config-volume
|
||||
type: bind
|
||||
source: ./config/messenger.yaml
|
||||
target: /etc/messenger/messenger.yaml
|
||||
read_only: true
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-alpine3.22
|
||||
@@ -34,6 +40,32 @@ services:
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
|
||||
redis-broker:
|
||||
image: redis:8.2.2-alpine
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --appendfsync
|
||||
- always
|
||||
- --maxmemory-policy
|
||||
- noeviction
|
||||
- --requirepass
|
||||
- ${REDIS_BROKER_PASSWORD}
|
||||
environment:
|
||||
REDIS_BROKER_PASSWORD: ${REDIS_BROKER_PASSWORD}
|
||||
volumes:
|
||||
- redis_broker_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_BROKER_PASSWORD\" ping | grep -q PONG"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: ${REDIS_BROKER_CPUS}
|
||||
mem_limit: ${REDIS_BROKER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
|
||||
s3:
|
||||
build:
|
||||
context: .
|
||||
@@ -89,14 +121,10 @@ services:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -111,42 +139,19 @@ services:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
APP_HOST: ${APP_HOST}
|
||||
APP_PORT: ${APP_PORT}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
CACHE_URL: ${CACHE_URL}
|
||||
SESSION_STORE: ${SESSION_STORE}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
|
||||
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
|
||||
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
|
||||
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
|
||||
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
|
||||
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
|
||||
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
|
||||
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
|
||||
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
||||
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
||||
SWAGGER_PORT: ${SWAGGER_PORT}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_USE_TLS: ${S3_USE_TLS}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||
ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES}
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
@@ -170,6 +175,66 @@ services:
|
||||
restart: unless-stopped
|
||||
networks: [application, data]
|
||||
|
||||
dispatcher:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/messenger", "dispatch"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
expose:
|
||||
- "9091"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis-broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: ${DISPATCHER_CPUS}
|
||||
mem_limit: ${DISPATCHER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data, monitoring]
|
||||
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/messenger", "worker"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
expose:
|
||||
- "9091"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis-broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: ${WORKER_CPUS}
|
||||
mem_limit: ${WORKER_MEMORY}
|
||||
stop_grace_period: 8s
|
||||
restart: unless-stopped
|
||||
networks: [data, monitoring]
|
||||
|
||||
nginx:
|
||||
build:
|
||||
context: .
|
||||
@@ -222,6 +287,32 @@ services:
|
||||
mem_limit: ${TEST_MEMORY}
|
||||
networks: [data]
|
||||
|
||||
lab5-experiment:
|
||||
profiles: ["experiments"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/lab5experiment"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis-broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
cpus: ${TEST_CPUS}
|
||||
mem_limit: ${TEST_MEMORY}
|
||||
networks: [application, data]
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
|
||||
environment:
|
||||
@@ -229,6 +320,9 @@ services:
|
||||
DATA_SOURCE_USER: ${POSTGRES_USER}
|
||||
DATA_SOURCE_PASS: ${POSTGRES_PASSWORD}
|
||||
PG_EXPORTER_COLLECTION_TIMEOUT: 5s
|
||||
PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres_exporter/queries.yml
|
||||
volumes:
|
||||
- ./monitoring/postgres-queries.yml:/etc/postgres_exporter/queries.yml:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -251,6 +345,21 @@ services:
|
||||
restart: unless-stopped
|
||||
networks: [data, monitoring]
|
||||
|
||||
redis-broker-exporter:
|
||||
image: oliver006/redis_exporter:v1.89.0-alpine
|
||||
environment:
|
||||
REDIS_ADDR: redis://redis-broker:6379
|
||||
REDIS_USER: default
|
||||
REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD}
|
||||
REDIS_EXPORTER_CHECK_SINGLE_STREAMS: attachment-jobs,attachment-jobs-failed
|
||||
depends_on:
|
||||
redis-broker:
|
||||
condition: service_healthy
|
||||
cpus: ${EXPORTER_CPUS}
|
||||
mem_limit: ${EXPORTER_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [data, monitoring]
|
||||
|
||||
nginx-exporter:
|
||||
image: nginx/nginx-prometheus-exporter:1.5.1
|
||||
command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"]
|
||||
@@ -285,6 +394,9 @@ services:
|
||||
- nginx-exporter
|
||||
- postgres-exporter
|
||||
- valkey-exporter
|
||||
- redis-broker-exporter
|
||||
- dispatcher
|
||||
- worker
|
||||
- s3
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
@@ -314,6 +426,7 @@ services:
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
redis_broker_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
s3_data:
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
version: 1
|
||||
|
||||
server:
|
||||
session_store: valkey
|
||||
session_ttl: 1h
|
||||
idempotency_ttl: 24h
|
||||
request_timeout: 5s
|
||||
http:
|
||||
read_header_timeout: 5s
|
||||
read_timeout: 10s
|
||||
write_timeout: 10s
|
||||
idle_timeout: 1m
|
||||
shutdown_timeout: 5s
|
||||
password:
|
||||
memory_kib: 65536
|
||||
iterations: 3
|
||||
parallelism: 2
|
||||
salt_length: 16
|
||||
key_length: 32
|
||||
|
||||
database:
|
||||
min_connections: 1
|
||||
max_connections: 10
|
||||
max_connection_age: 30m
|
||||
max_connection_idle: 5m
|
||||
health_check_period: 30s
|
||||
dependency_timeout: 5s
|
||||
|
||||
migration:
|
||||
timeout: 30s
|
||||
|
||||
object_storage:
|
||||
use_tls: false
|
||||
timeout: 5s
|
||||
|
||||
attachments:
|
||||
max_bytes: 10485760
|
||||
max_pixels: 40000000
|
||||
body_memory_bytes: 67108864
|
||||
decode_memory_bytes: 402653184
|
||||
|
||||
pipeline:
|
||||
version: 1
|
||||
|
||||
broker:
|
||||
stream: attachment-jobs
|
||||
consumer_group: attachment-workers
|
||||
failed_stream: attachment-jobs-failed
|
||||
failed_stream_max_length: 10000
|
||||
|
||||
outbox:
|
||||
batch_size: 32
|
||||
lease: 15s
|
||||
poll_interval: 1s
|
||||
recovery_interval: 10s
|
||||
retry_delay: 2s
|
||||
retention: 168h
|
||||
cleanup_interval: 1m
|
||||
cleanup_batch_size: 1000
|
||||
|
||||
worker:
|
||||
concurrency: 2
|
||||
prefetch: 4
|
||||
lease: 30s
|
||||
heartbeat_interval: 5s
|
||||
processing_timeout: 20s
|
||||
read_block: 2s
|
||||
reclaim_interval: 5s
|
||||
max_attempts: 5
|
||||
retry:
|
||||
base: 1s
|
||||
max: 30s
|
||||
memory_bytes: 738197504
|
||||
|
||||
monitoring:
|
||||
metrics_address: 0.0.0.0:9091
|
||||
@@ -5,6 +5,12 @@ services:
|
||||
migrate:
|
||||
image: ${API_IMAGE:?API_IMAGE is required}
|
||||
|
||||
dispatcher:
|
||||
image: ${API_IMAGE:?API_IMAGE is required}
|
||||
|
||||
worker:
|
||||
image: ${API_IMAGE:?API_IMAGE is required}
|
||||
|
||||
nginx:
|
||||
image: ${NGINX_IMAGE:?NGINX_IMAGE is required}
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"started_at": "2026-10-22T14:19:25Z",
|
||||
"duration_ms": 6981,
|
||||
"pipeline_version": 1,
|
||||
"fault_scenarios": [
|
||||
{
|
||||
"name": "before_side_effect",
|
||||
"attempt_after_recovery": 2,
|
||||
"objects_before_recovery": 0,
|
||||
"objects_after_recovery": 4,
|
||||
"final_status": "ready"
|
||||
},
|
||||
{
|
||||
"name": "after_side_effect",
|
||||
"attempt_after_recovery": 2,
|
||||
"objects_before_recovery": 4,
|
||||
"objects_after_recovery": 4,
|
||||
"final_status": "ready"
|
||||
},
|
||||
{
|
||||
"name": "after_commit",
|
||||
"attempt_after_recovery": 1,
|
||||
"objects_before_recovery": 4,
|
||||
"objects_after_recovery": 4,
|
||||
"final_status": "ready"
|
||||
}
|
||||
],
|
||||
"duplicate_deliveries": 2,
|
||||
"v1_object_count": 4,
|
||||
"v2_object_count": 8,
|
||||
"accepted_jobs": 5,
|
||||
"terminal_jobs": 5,
|
||||
"ready_jobs": 4,
|
||||
"failed_jobs": 1,
|
||||
"permanent_error_code": "invalid_image",
|
||||
"replay_attempt": 2,
|
||||
"invariant_results": {
|
||||
"accepted_jobs_reach_terminal_state": true,
|
||||
"duplicate_effect_is_unique": true,
|
||||
"old_pipeline_cannot_replace_new": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"accepted_before_outbox_publish":true,"accepted_while_worker_stopped":true,"content_conflict_while_queued":true,"linked_while_queued":true,"original_preserved":true,"ready_after_api_broker_restart":true,"thumbnail_dimensions":"256x160"}
|
||||
@@ -3,10 +3,12 @@ module gitea.privcloud.tech/dizi/mtusi2026lab5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/deepteams/webp v1.2.8
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.0.99
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
@@ -18,6 +20,7 @@ require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA=
|
||||
github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
@@ -64,6 +72,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
)
|
||||
|
||||
func RunDispatcher(logger *slog.Logger) error {
|
||||
settings, err := config.LoadDispatcher()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
client, err := newRedisClient(settings.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher",
|
||||
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength,
|
||||
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.Ensure(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
||||
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
||||
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
||||
RetryDelay: settings.RetryDelay, Retention: settings.Retention,
|
||||
CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("outbox dispatcher started", "stream", settings.Stream)
|
||||
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run)
|
||||
}
|
||||
|
||||
func RunWorker(logger *slog.Logger) error {
|
||||
settings, err := config.LoadWorker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := newRedisClient(settings.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
consumer := hostname + "-" + uuid.NewString()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
||||
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock,
|
||||
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.Ensure(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
processorWorker, err := worker.New(
|
||||
logger, postgres.New(pool), broker, objects, processor, metrics,
|
||||
worker.Config{
|
||||
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
|
||||
HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout,
|
||||
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
|
||||
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
MemoryBytes: settings.MemoryBytes,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency)
|
||||
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run)
|
||||
}
|
||||
|
||||
func newRedisClient(rawURL string) (*redis.Client, error) {
|
||||
options, err := redis.ParseURL(rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := redis.NewClient(options)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
_ = client.Close()
|
||||
return nil, err
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func runPipelineProcess(
|
||||
logger *slog.Logger,
|
||||
metricsAddress string,
|
||||
metricsHandler http.Handler,
|
||||
run func(context.Context) error,
|
||||
) error {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", metricsHandler)
|
||||
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
||||
server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
||||
listener, err := net.Listen("tcp", metricsAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serveResult := make(chan error, 1)
|
||||
go func() { serveResult <- server.Serve(listener) }()
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
runResult := make(chan error, 1)
|
||||
go func() { runResult <- run(ctx) }()
|
||||
|
||||
var result error
|
||||
select {
|
||||
case result = <-runResult:
|
||||
stop()
|
||||
case result = <-serveResult:
|
||||
stop()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownCtx); err != nil && result == nil {
|
||||
result = err
|
||||
}
|
||||
if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
logger.Info("pipeline process stopped")
|
||||
return result
|
||||
}
|
||||
|
||||
func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config {
|
||||
return database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func workerDatabaseConfig(settings config.WorkerConfig) database.Config {
|
||||
return database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,7 @@ func RunReconciliation(arguments []string, output io.Writer) error {
|
||||
}
|
||||
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
|
||||
OrphanPrefixes: []string{"staging/"},
|
||||
UnreferencedRetention: *retention, Apply: *apply,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
)
|
||||
|
||||
func RunAttachmentReplay(arguments []string, output io.Writer) error {
|
||||
if len(arguments) != 3 {
|
||||
return errors.New("usage: messenger replay-attachment <attachment-id> <actor> <reason>")
|
||||
}
|
||||
settings, err := config.LoadJobAdmin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
job, err := postgres.New(pool).ReplayFailedAttachment(
|
||||
context.Background(), arguments[0], arguments[1], arguments[2],
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt)
|
||||
return err
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func RunServer(logger *slog.Logger) error {
|
||||
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
|
||||
CleanupTimeout: settings.RequestTimeout,
|
||||
CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
)
|
||||
|
||||
const (
|
||||
payloadField = "payload"
|
||||
failedPayloadMaxBytes = 64 << 10
|
||||
)
|
||||
|
||||
var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
|
||||
|
||||
var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer")
|
||||
|
||||
var _ outbox.Publisher = (*Broker)(nil)
|
||||
|
||||
type Config struct {
|
||||
Stream string
|
||||
Group string
|
||||
Consumer string
|
||||
FailedStream string
|
||||
FailedMaxLen int64
|
||||
Block time.Duration
|
||||
ClaimMinIdle time.Duration
|
||||
ReadCount int64
|
||||
}
|
||||
|
||||
type Broker struct {
|
||||
client redis.UniversalClient
|
||||
config Config
|
||||
}
|
||||
|
||||
type Delivery struct {
|
||||
ID string
|
||||
Envelope Envelope
|
||||
DecodeError error
|
||||
|
||||
broker *Broker
|
||||
payload []byte
|
||||
}
|
||||
|
||||
type FailedDelivery struct {
|
||||
Code string
|
||||
FailedAt time.Time
|
||||
}
|
||||
|
||||
func New(client redis.UniversalClient, config Config) (*Broker, error) {
|
||||
if client == nil {
|
||||
return nil, errors.New("redis client is required")
|
||||
}
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Broker{client: client, config: config}, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case c.Stream == "":
|
||||
return errors.New("stream is required")
|
||||
case c.Group == "":
|
||||
return errors.New("consumer group is required")
|
||||
case c.Consumer == "":
|
||||
return errors.New("consumer name is required")
|
||||
case c.FailedStream == "":
|
||||
return errors.New("failed stream is required")
|
||||
case c.FailedStream == c.Stream:
|
||||
return errors.New("failed stream must differ from job stream")
|
||||
case c.FailedMaxLen <= 0:
|
||||
return errors.New("failed stream maximum length must be positive")
|
||||
case c.Block < 0:
|
||||
return errors.New("block duration must not be negative")
|
||||
case c.ClaimMinIdle <= 0:
|
||||
return errors.New("claim minimum idle time must be positive")
|
||||
case c.ReadCount <= 0:
|
||||
return errors.New("read count must be positive")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Broker) Ensure(ctx context.Context) error {
|
||||
err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err()
|
||||
if err != nil && !isBusyGroup(err) {
|
||||
return fmt.Errorf("create redis stream consumer group: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isBusyGroup(err error) bool {
|
||||
return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP")
|
||||
}
|
||||
|
||||
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
|
||||
envelope, err := DecodeEnvelope(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
payload, err = EncodeEnvelope(envelope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
id, err := b.client.XAdd(ctx, &redis.XAddArgs{
|
||||
Stream: b.config.Stream,
|
||||
Values: map[string]any{payloadField: string(payload)},
|
||||
}).Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("publish redis stream job: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
|
||||
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: b.config.Group,
|
||||
Consumer: b.config.Consumer,
|
||||
Streams: []string{b.config.Stream, ">"},
|
||||
Count: b.config.ReadCount,
|
||||
Block: b.config.Block,
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read redis stream jobs: %w", err)
|
||||
}
|
||||
return b.deliveries(streams)
|
||||
}
|
||||
|
||||
func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) {
|
||||
messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||
Stream: b.config.Stream,
|
||||
Group: b.config.Group,
|
||||
Consumer: b.config.Consumer,
|
||||
MinIdle: b.config.ClaimMinIdle,
|
||||
Start: start,
|
||||
Count: b.config.ReadCount,
|
||||
}).Result()
|
||||
if err != nil {
|
||||
return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err)
|
||||
}
|
||||
|
||||
deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}})
|
||||
if err != nil {
|
||||
return nil, start, err
|
||||
}
|
||||
return deliveries, next, nil
|
||||
}
|
||||
|
||||
func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) {
|
||||
var deliveries []Delivery
|
||||
for _, stream := range streams {
|
||||
for _, message := range stream.Messages {
|
||||
delivery := Delivery{ID: message.ID, broker: b}
|
||||
payload, err := messagePayload(message)
|
||||
if err != nil {
|
||||
delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
delivery.payload = boundedPayload(payload)
|
||||
envelope, err := DecodeEnvelope(payload)
|
||||
if err != nil {
|
||||
delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
delivery.Envelope = envelope
|
||||
delivery.payload, err = EncodeEnvelope(envelope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err)
|
||||
}
|
||||
deliveries = append(deliveries, delivery)
|
||||
}
|
||||
}
|
||||
return deliveries, nil
|
||||
}
|
||||
|
||||
func boundedPayload(payload []byte) []byte {
|
||||
if len(payload) > failedPayloadMaxBytes {
|
||||
payload = payload[:failedPayloadMaxBytes]
|
||||
}
|
||||
return append([]byte(nil), payload...)
|
||||
}
|
||||
|
||||
func messagePayload(message redis.XMessage) ([]byte, error) {
|
||||
value, ok := message.Values[payloadField]
|
||||
if !ok {
|
||||
return nil, errors.New("payload field is missing")
|
||||
}
|
||||
switch payload := value.(type) {
|
||||
case string:
|
||||
return []byte(payload), nil
|
||||
case []byte:
|
||||
return payload, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("payload has type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func (d Delivery) Ack(ctx context.Context) error {
|
||||
if d.broker == nil {
|
||||
return errors.New("delivery is detached from broker")
|
||||
}
|
||||
acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream},
|
||||
d.broker.config.Group, d.ID, d.broker.config.Consumer,
|
||||
).Int64()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ack redis stream job: %w", err)
|
||||
}
|
||||
if acked == 0 {
|
||||
return ErrDeliveryNotOwned
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const ackOwnedScript = `
|
||||
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
||||
if #pending == 0 or pending[1][2] ~= ARGV[3] then
|
||||
return 0
|
||||
end
|
||||
local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
||||
if acked == 1 then
|
||||
redis.call('XDEL', KEYS[1], ARGV[2])
|
||||
end
|
||||
return acked
|
||||
`
|
||||
|
||||
func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error {
|
||||
if d.broker == nil {
|
||||
return errors.New("delivery is detached from broker")
|
||||
}
|
||||
if !failureCodePattern.MatchString(failure.Code) {
|
||||
return errors.New("failure code is invalid")
|
||||
}
|
||||
if failure.FailedAt.IsZero() {
|
||||
return errors.New("failure time is required")
|
||||
}
|
||||
|
||||
payload := d.payload
|
||||
if d.DecodeError == nil && len(payload) == 0 {
|
||||
var err error
|
||||
payload, err = EncodeEnvelope(d.Envelope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
moved, err := d.broker.client.Eval(ctx, moveToFailedScript,
|
||||
[]string{d.broker.config.FailedStream, d.broker.config.Stream},
|
||||
d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code,
|
||||
failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt),
|
||||
strconv.FormatInt(d.broker.config.FailedMaxLen, 10),
|
||||
).Int64()
|
||||
if err != nil {
|
||||
return fmt.Errorf("move redis stream job to failed stream: %w", err)
|
||||
}
|
||||
if moved == 0 {
|
||||
return ErrDeliveryNotOwned
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const moveToFailedScript = `
|
||||
local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1)
|
||||
if #pending == 0 or pending[1][2] ~= ARGV[3] then
|
||||
return 0
|
||||
end
|
||||
redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*',
|
||||
'payload', ARGV[4],
|
||||
'source_message_id', ARGV[2],
|
||||
'error_code', ARGV[5],
|
||||
'failed_at', ARGV[6],
|
||||
'attempt', ARGV[7])
|
||||
local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2])
|
||||
if acked == 1 then
|
||||
redis.call('XDEL', KEYS[2], ARGV[2])
|
||||
end
|
||||
return acked
|
||||
`
|
||||
@@ -0,0 +1,93 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBrokerDecodesDeliveries(t *testing.T) {
|
||||
payload, err := EncodeEnvelope(validEnvelope())
|
||||
require.NoError(t, err)
|
||||
|
||||
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
|
||||
deliveries, err := broker.deliveries([]redis.XStream{{
|
||||
Stream: "attachment-jobs",
|
||||
Messages: []redis.XMessage{{
|
||||
ID: "1-0",
|
||||
Values: map[string]any{payloadField: string(payload)},
|
||||
}},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deliveries, 1)
|
||||
require.Equal(t, "1-0", deliveries[0].ID)
|
||||
require.Equal(t, validEnvelope(), deliveries[0].Envelope)
|
||||
}
|
||||
|
||||
func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) {
|
||||
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
|
||||
deliveries, err := broker.deliveries([]redis.XStream{{
|
||||
Stream: "attachment-jobs",
|
||||
Messages: []redis.XMessage{{
|
||||
ID: "1-0",
|
||||
Values: map[string]any{payloadField: `{
|
||||
"schema_version":1,
|
||||
"job_id":"11111111-1111-4111-8111-111111111111",
|
||||
"type":"attachment.process.v1",
|
||||
"attachment_id":"22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version":1,
|
||||
"attempt":1,
|
||||
"created_at":"2026-09-01T12:00:00Z",
|
||||
"storage_key":"private/object"
|
||||
}`},
|
||||
}},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deliveries, 1)
|
||||
require.Error(t, deliveries[0].DecodeError)
|
||||
require.Contains(t, string(deliveries[0].payload), `"storage_key":"private/object"`)
|
||||
}
|
||||
|
||||
func TestBrokerBoundsMalformedPayloadForFailedStream(t *testing.T) {
|
||||
payload := make([]byte, failedPayloadMaxBytes+1)
|
||||
for index := range payload {
|
||||
payload[index] = 'x'
|
||||
}
|
||||
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
|
||||
deliveries, err := broker.deliveries([]redis.XStream{{
|
||||
Stream: "attachment-jobs",
|
||||
Messages: []redis.XMessage{{
|
||||
ID: "1-0",
|
||||
Values: map[string]any{payloadField: payload},
|
||||
}},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deliveries, 1)
|
||||
require.Error(t, deliveries[0].DecodeError)
|
||||
require.Len(t, deliveries[0].payload, failedPayloadMaxBytes)
|
||||
}
|
||||
|
||||
func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) {
|
||||
broker := &Broker{}
|
||||
_, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`))
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestConfigValidation(t *testing.T) {
|
||||
config := Config{
|
||||
Stream: "attachment-jobs",
|
||||
Group: "attachment-workers",
|
||||
Consumer: "worker-1",
|
||||
FailedStream: "attachment-jobs-failed",
|
||||
FailedMaxLen: 1000,
|
||||
Block: time.Second,
|
||||
ClaimMinIdle: time.Minute,
|
||||
ReadCount: 10,
|
||||
}
|
||||
require.NoError(t, config.validate())
|
||||
|
||||
config.ClaimMinIdle = 0
|
||||
require.Error(t, config.validate())
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
SchemaVersion = 1
|
||||
JobType = "attachment.process.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidEnvelope = errors.New("invalid job envelope")
|
||||
traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
|
||||
uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
)
|
||||
|
||||
type Envelope struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID uuid.UUID `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
func (e Envelope) Validate() error {
|
||||
switch {
|
||||
case e.SchemaVersion != SchemaVersion:
|
||||
return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope)
|
||||
case e.Type != JobType:
|
||||
return fmt.Errorf("%w: unsupported type", errInvalidEnvelope)
|
||||
case e.JobID == uuid.Nil:
|
||||
return fmt.Errorf("%w: job_id is required", errInvalidEnvelope)
|
||||
case e.AttachmentID == uuid.Nil:
|
||||
return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope)
|
||||
case e.PipelineVersion < 1:
|
||||
return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope)
|
||||
case e.Attempt < 1:
|
||||
return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope)
|
||||
case e.CreatedAt.IsZero():
|
||||
return fmt.Errorf("%w: created_at is required", errInvalidEnvelope)
|
||||
case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent):
|
||||
return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope)
|
||||
case utf8.RuneCountInString(e.Tracestate) > 512:
|
||||
return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func EncodeEnvelope(envelope Envelope) ([]byte, error) {
|
||||
if err := envelope.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode job envelope: %w", err)
|
||||
}
|
||||
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func DecodeEnvelope(payload []byte) (Envelope, error) {
|
||||
var wire envelopeJSON
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&wire); err != nil {
|
||||
return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err)
|
||||
}
|
||||
if err := ensureJSONEnd(decoder); err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
|
||||
jobID, err := parseUUID(wire.JobID, "job_id")
|
||||
if err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id")
|
||||
if err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt)
|
||||
if err != nil {
|
||||
return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope)
|
||||
}
|
||||
envelope := Envelope{
|
||||
SchemaVersion: wire.SchemaVersion,
|
||||
JobID: jobID,
|
||||
Type: wire.Type,
|
||||
AttachmentID: attachmentID,
|
||||
PipelineVersion: wire.PipelineVersion,
|
||||
Attempt: wire.Attempt,
|
||||
CreatedAt: createdAt,
|
||||
Traceparent: wire.Traceparent,
|
||||
Tracestate: wire.Tracestate,
|
||||
}
|
||||
if err := envelope.Validate(); err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
|
||||
return envelope, nil
|
||||
}
|
||||
|
||||
type envelopeJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
func parseUUID(value, field string) (uuid.UUID, error) {
|
||||
if !uuidPattern.MatchString(value) {
|
||||
return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field)
|
||||
}
|
||||
parsed, err := uuid.Parse(value)
|
||||
if err != nil {
|
||||
return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func ensureJSONEnd(decoder *json.Decoder) error {
|
||||
var value any
|
||||
if err := decoder.Decode(&value); !errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEnvelopeRoundTrip(t *testing.T) {
|
||||
envelope := validEnvelope()
|
||||
payload, err := EncodeEnvelope(envelope)
|
||||
require.NoError(t, err)
|
||||
|
||||
decoded, err := DecodeEnvelope(payload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, envelope, decoded)
|
||||
}
|
||||
|
||||
func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"schema_version":1,
|
||||
"job_id":"11111111-1111-4111-8111-111111111111",
|
||||
"type":"attachment.process.v1",
|
||||
"attachment_id":"22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version":1,
|
||||
"attempt":1,
|
||||
"created_at":"2026-09-01T12:00:00Z",
|
||||
"storage_key":"must-not-enter-the-broker"
|
||||
}`)
|
||||
|
||||
_, err := DecodeEnvelope(payload)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestEnvelopeValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Envelope)
|
||||
}{
|
||||
{name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }},
|
||||
{name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }},
|
||||
{name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }},
|
||||
{name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }},
|
||||
{name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }},
|
||||
{name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }},
|
||||
{name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }},
|
||||
{name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }},
|
||||
{name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
envelope := validEnvelope()
|
||||
tt.mutate(&envelope)
|
||||
require.Error(t, envelope.Validate())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validEnvelope() Envelope {
|
||||
return Envelope{
|
||||
SchemaVersion: SchemaVersion,
|
||||
JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"),
|
||||
Type: JobType,
|
||||
AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"),
|
||||
PipelineVersion: 1,
|
||||
Attempt: 1,
|
||||
CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC),
|
||||
Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
||||
Tracestate: "vendor=value",
|
||||
}
|
||||
}
|
||||
+66
-50
@@ -46,6 +46,7 @@ type ServerConfig struct {
|
||||
AttachmentMaxPixels int64
|
||||
AttachmentBodyMemory int64
|
||||
AttachmentDecodeMemory int64
|
||||
PipelineVersion int
|
||||
}
|
||||
|
||||
type MigrationConfig struct {
|
||||
@@ -97,14 +98,16 @@ type objectStorageConfig struct {
|
||||
}
|
||||
|
||||
func LoadServer() (ServerConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
|
||||
host, err := requiredString(values, "APP_HOST")
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
port, err := requiredPort(values, "APP_PORT")
|
||||
|
||||
host, err := requiredString(values, "server.host")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
port, err := requiredPort(values, "server.port")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
@@ -112,71 +115,71 @@ func LoadServer() (ServerConfig, error) {
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
cacheURL, err := requiredString(values, "CACHE_URL")
|
||||
cacheURL, err := requiredString(values, "cache.url")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
store, err := requiredString(values, "SESSION_STORE")
|
||||
store, err := requiredString(values, "server.session_store")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
if store != "valkey" {
|
||||
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
|
||||
return ServerConfig{}, errors.New("server.session_store must be valkey")
|
||||
}
|
||||
|
||||
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
|
||||
sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
|
||||
idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
|
||||
requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
|
||||
readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
|
||||
readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
|
||||
writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
|
||||
idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
|
||||
shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
|
||||
passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
|
||||
passwordIterations, err := integer(values, "server.password.iterations", 1, 20)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
|
||||
passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
|
||||
passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
|
||||
passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
|
||||
swaggerPort, err := requiredPort(values, "server.swagger_port")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
@@ -184,28 +187,32 @@ func LoadServer() (ServerConfig, error) {
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
|
||||
return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes)
|
||||
}
|
||||
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
|
||||
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
|
||||
return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory)
|
||||
}
|
||||
pipelineVersion, err := integer(values, "pipeline.version", 1, 1000)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
|
||||
return ServerConfig{
|
||||
@@ -226,17 +233,20 @@ func LoadServer() (ServerConfig, error) {
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
AttachmentBodyMemory: int64(attachmentBodyMemory),
|
||||
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
|
||||
PipelineVersion: pipelineVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadMigration() (MigrationConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
|
||||
migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute)
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
@@ -249,8 +259,10 @@ func LoadMigration() (MigrationConfig, error) {
|
||||
}
|
||||
|
||||
func LoadReconciliation() (ReconciliationConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return ReconciliationConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return ReconciliationConfig{}, err
|
||||
@@ -270,31 +282,31 @@ func LoadReconciliation() (ReconciliationConfig, error) {
|
||||
}
|
||||
|
||||
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
|
||||
endpoint, err := requiredString(values, "S3_ENDPOINT")
|
||||
endpoint, err := requiredString(values, "object_storage.endpoint")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
region, err := requiredString(values, "S3_REGION")
|
||||
region, err := requiredString(values, "object_storage.region")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
bucket, err := requiredString(values, "S3_BUCKET")
|
||||
bucket, err := requiredString(values, "object_storage.bucket")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
|
||||
accessKey, err := requiredString(values, "object_storage.access_key")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
secretKey, err := requiredString(values, "S3_SECRET_KEY")
|
||||
secretKey, err := requiredString(values, "object_storage.secret_key")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
useTLS, err := boolean(values, "S3_USE_TLS")
|
||||
useTLS, err := boolean(values, "object_storage.use_tls")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
|
||||
timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
@@ -305,31 +317,31 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
|
||||
}
|
||||
|
||||
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
|
||||
databaseURL, err := requiredString(values, "DATABASE_URL")
|
||||
databaseURL, err := requiredString(values, "database.url")
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
|
||||
maxConns, err := integer(values, "database.max_connections", 1, 100)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
|
||||
minConns, err := integer(values, "database.min_connections", 0, maxConns)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
|
||||
maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
|
||||
maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
|
||||
healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
|
||||
dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
@@ -359,12 +371,16 @@ func requiredPort(values *viper.Viper, name string) (string, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) {
|
||||
value, err := integer(values, name, minimum, maximum)
|
||||
func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) {
|
||||
value, err := requiredString(values, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(value) * time.Second, nil
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil || parsed < minimum || parsed > maximum {
|
||||
return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) {
|
||||
|
||||
+114
-12
@@ -1,24 +1,17 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
|
||||
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
|
||||
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
|
||||
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
|
||||
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
|
||||
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
|
||||
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
|
||||
}
|
||||
for name, value := range values {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
setConfigEnvironment(t, validTestConfig)
|
||||
t.Setenv("APP_HOST", "")
|
||||
t.Setenv("CACHE_URL", "")
|
||||
t.Setenv("SWAGGER_PORT", "")
|
||||
@@ -27,3 +20,112 @@ func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "attachments", settings.S3Bucket)
|
||||
}
|
||||
|
||||
func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) {
|
||||
setConfigEnvironment(t, validTestConfig)
|
||||
t.Setenv("WORKER_CONCURRENCY", "64")
|
||||
|
||||
settings, err := LoadWorker()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, settings.Concurrency)
|
||||
require.Equal(t, 5*time.Second, settings.HeartbeatInterval)
|
||||
}
|
||||
|
||||
func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) {
|
||||
content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1)
|
||||
setConfigEnvironment(t, content)
|
||||
|
||||
_, err := LoadWorker()
|
||||
require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease")
|
||||
}
|
||||
|
||||
func TestLoadValuesRejectsUnknownVersion(t *testing.T) {
|
||||
setConfigEnvironment(t, "version: 2\n")
|
||||
|
||||
_, err := loadValues()
|
||||
require.ErrorContains(t, err, "version: must be 1")
|
||||
}
|
||||
|
||||
func setConfigEnvironment(t *testing.T, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "messenger.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
|
||||
t.Setenv("MESSENGER_CONFIG_PATH", path)
|
||||
for name, value := range map[string]string{
|
||||
"APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081",
|
||||
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
|
||||
"CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0",
|
||||
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
|
||||
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
|
||||
} {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
}
|
||||
|
||||
const validTestConfig = `version: 1
|
||||
server:
|
||||
session_store: valkey
|
||||
session_ttl: 1h
|
||||
idempotency_ttl: 24h
|
||||
request_timeout: 5s
|
||||
http:
|
||||
read_header_timeout: 5s
|
||||
read_timeout: 10s
|
||||
write_timeout: 10s
|
||||
idle_timeout: 1m
|
||||
shutdown_timeout: 5s
|
||||
password:
|
||||
memory_kib: 65536
|
||||
iterations: 3
|
||||
parallelism: 2
|
||||
salt_length: 16
|
||||
key_length: 32
|
||||
database:
|
||||
min_connections: 0
|
||||
max_connections: 4
|
||||
max_connection_age: 5m
|
||||
max_connection_idle: 1m
|
||||
health_check_period: 30s
|
||||
dependency_timeout: 5s
|
||||
migration:
|
||||
timeout: 30s
|
||||
object_storage:
|
||||
use_tls: false
|
||||
timeout: 5s
|
||||
attachments:
|
||||
max_bytes: 10485760
|
||||
max_pixels: 40000000
|
||||
body_memory_bytes: 67108864
|
||||
decode_memory_bytes: 402653184
|
||||
pipeline:
|
||||
version: 1
|
||||
broker:
|
||||
stream: attachment-jobs
|
||||
consumer_group: attachment-workers
|
||||
failed_stream: attachment-jobs-failed
|
||||
failed_stream_max_length: 10000
|
||||
outbox:
|
||||
batch_size: 32
|
||||
lease: 15s
|
||||
poll_interval: 1s
|
||||
recovery_interval: 10s
|
||||
retry_delay: 2s
|
||||
retention: 168h
|
||||
cleanup_interval: 1m
|
||||
cleanup_batch_size: 1000
|
||||
worker:
|
||||
concurrency: 2
|
||||
prefetch: 4
|
||||
lease: 30s
|
||||
heartbeat_interval: 5s
|
||||
processing_timeout: 20s
|
||||
read_block: 2s
|
||||
reclaim_interval: 5s
|
||||
max_attempts: 5
|
||||
retry:
|
||||
base: 1s
|
||||
max: 30s
|
||||
memory_bytes: 738197504
|
||||
monitoring:
|
||||
metrics_address: 0.0.0.0:9091
|
||||
`
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
const supportedConfigVersion = 1
|
||||
|
||||
var environmentKeys = map[string]string{
|
||||
"server.host": "APP_HOST",
|
||||
"server.port": "APP_PORT",
|
||||
"server.swagger_port": "SWAGGER_PORT",
|
||||
"database.url": "DATABASE_URL",
|
||||
"cache.url": "CACHE_URL",
|
||||
"object_storage.endpoint": "S3_ENDPOINT",
|
||||
"object_storage.region": "S3_REGION",
|
||||
"object_storage.bucket": "S3_BUCKET",
|
||||
"object_storage.access_key": "S3_ACCESS_KEY",
|
||||
"object_storage.secret_key": "S3_SECRET_KEY",
|
||||
"broker.url": "BROKER_URL",
|
||||
}
|
||||
|
||||
func loadValues() (*viper.Viper, error) {
|
||||
path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH"))
|
||||
if path == "" {
|
||||
return nil, errors.New("MESSENGER_CONFIG_PATH is required")
|
||||
}
|
||||
values := viper.New()
|
||||
values.SetConfigFile(path)
|
||||
if err := values.ReadInConfig(); err != nil {
|
||||
return nil, fmt.Errorf("read messenger config: %w", err)
|
||||
}
|
||||
if version := values.GetInt("version"); version != supportedConfigVersion {
|
||||
return nil, fmt.Errorf("version: must be %d", supportedConfigVersion)
|
||||
}
|
||||
for key, name := range environmentKeys {
|
||||
if err := values.BindEnv(key, name); err != nil {
|
||||
return nil, fmt.Errorf("bind %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
type DispatcherConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
BrokerURL string
|
||||
Stream string
|
||||
ConsumerGroup string
|
||||
FailedStream string
|
||||
FailedStreamMaxLength int64
|
||||
BatchSize int
|
||||
LeaseDuration time.Duration
|
||||
PollInterval time.Duration
|
||||
RecoveryInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
Retention time.Duration
|
||||
CleanupInterval time.Duration
|
||||
CleanupBatchSize int
|
||||
MetricsAddress string
|
||||
}
|
||||
|
||||
type WorkerConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3UseTLS bool
|
||||
S3Timeout time.Duration
|
||||
BrokerURL string
|
||||
Stream string
|
||||
ConsumerGroup string
|
||||
FailedStream string
|
||||
FailedStreamMaxLength int64
|
||||
Concurrency int
|
||||
Prefetch int
|
||||
LeaseDuration time.Duration
|
||||
HeartbeatInterval time.Duration
|
||||
ProcessingTimeout time.Duration
|
||||
ReadBlock time.Duration
|
||||
ReclaimInterval time.Duration
|
||||
MaxAttempts int
|
||||
RetryBase time.Duration
|
||||
RetryMax time.Duration
|
||||
AttachmentMaxBytes int64
|
||||
AttachmentMaxPixels int64
|
||||
MemoryBytes int64
|
||||
MetricsAddress string
|
||||
}
|
||||
|
||||
type JobAdminConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
}
|
||||
|
||||
type brokerConfig struct {
|
||||
url string
|
||||
stream string
|
||||
consumerGroup string
|
||||
failedStream string
|
||||
failedStreamMaxLength int
|
||||
}
|
||||
|
||||
func LoadJobAdmin() (JobAdminConfig, error) {
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return JobAdminConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return JobAdminConfig{}, err
|
||||
}
|
||||
return JobAdminConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadDispatcher() (DispatcherConfig, error) {
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
broker, err := loadBrokerConfig(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
batchSize, err := integer(values, "outbox.batch_size", 1, 1000)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
metricsAddress, err := metricsAddress(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
return DispatcherConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
|
||||
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
||||
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
|
||||
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay,
|
||||
Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize,
|
||||
MetricsAddress: metricsAddress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadWorker() (WorkerConfig, error) {
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
storage, err := loadObjectStorageConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
broker, err := loadBrokerConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
concurrency, err := integer(values, "worker.concurrency", 1, 64)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
prefetch, err := integer(values, "worker.prefetch", concurrency, 1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if heartbeatInterval >= leaseDuration {
|
||||
return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease")
|
||||
}
|
||||
processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if processingTimeout >= leaseDuration {
|
||||
return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease")
|
||||
}
|
||||
readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
maxAttempts, err := integer(values, "worker.max_attempts", 1, 100)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if retryMax < retryBase {
|
||||
return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base")
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
metricsAddress, err := metricsAddress(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
return WorkerConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
||||
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS,
|
||||
S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream,
|
||||
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
|
||||
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
||||
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
|
||||
HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
|
||||
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func metricsAddress(values *viper.Viper) (string, error) {
|
||||
address, err := requiredString(values, "monitoring.metrics_address")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
|
||||
}
|
||||
if _, err := integerValue(port, 1, 65535); err != nil {
|
||||
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
|
||||
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
|
||||
url, err := requiredString(values, "broker.url")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
stream, err := requiredString(values, "broker.stream")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
consumerGroup, err := requiredString(values, "broker.consumer_group")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
failedStream, err := requiredString(values, "broker.failed_stream")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000)
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
return brokerConfig{
|
||||
url: url, stream: stream, consumerGroup: consumerGroup,
|
||||
failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60}
|
||||
|
||||
type Metrics struct {
|
||||
workerInFlight prometheus.Gauge
|
||||
workerJobs *prometheus.CounterVec
|
||||
workerDuration *prometheus.HistogramVec
|
||||
workerRetries prometheus.Counter
|
||||
workerReclaimed prometheus.Counter
|
||||
outboxPublished prometheus.Counter
|
||||
outboxPublishFailure prometheus.Counter
|
||||
outboxRecovered prometheus.Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func NewMetrics() *Metrics {
|
||||
registry := prometheus.NewRegistry()
|
||||
metrics := &Metrics{
|
||||
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
|
||||
Help: "Attachment jobs currently processed by this worker.",
|
||||
}),
|
||||
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
|
||||
Help: "Completed attachment job deliveries by result.",
|
||||
}, []string{"result"}),
|
||||
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
|
||||
Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets,
|
||||
}, []string{"result"}),
|
||||
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
|
||||
Help: "Attachment jobs scheduled for another attempt.",
|
||||
}),
|
||||
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
|
||||
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
|
||||
}),
|
||||
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
|
||||
Help: "Outbox events confirmed by Redis Streams.",
|
||||
}),
|
||||
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
|
||||
Help: "Outbox events whose Redis Streams publish failed.",
|
||||
}),
|
||||
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
|
||||
Help: "Expired outbox leases returned for publication.",
|
||||
}),
|
||||
}
|
||||
registry.MustRegister(
|
||||
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
|
||||
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
|
||||
metrics.outboxPublishFailure, metrics.outboxRecovered,
|
||||
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||
return metrics
|
||||
}
|
||||
|
||||
func (m *Metrics) Handler() http.Handler {
|
||||
return m.handler
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveStarted() {
|
||||
m.workerInFlight.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
|
||||
m.workerInFlight.Dec()
|
||||
m.workerJobs.WithLabelValues(result).Inc()
|
||||
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRetry() {
|
||||
m.workerRetries.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveReclaimed(count int) {
|
||||
m.workerReclaimed.Add(float64(count))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublished() {
|
||||
m.outboxPublished.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublishFailure() {
|
||||
m.outboxPublishFailure.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRecovered(count int) {
|
||||
m.outboxRecovered.Add(float64(count))
|
||||
}
|
||||
@@ -25,10 +25,10 @@ import (
|
||||
type Repository interface {
|
||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
|
||||
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
|
||||
QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error)
|
||||
DeletePendingAttachment(context.Context, string, string) error
|
||||
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
|
||||
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
|
||||
AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
@@ -43,6 +43,7 @@ type Config struct {
|
||||
BodyMemoryBytes int64
|
||||
DecodeMemoryBytes int64
|
||||
CleanupTimeout time.Duration
|
||||
PipelineVersion int
|
||||
}
|
||||
|
||||
type Upload struct {
|
||||
@@ -52,10 +53,13 @@ type Upload struct {
|
||||
ContentType string
|
||||
ContentLength int64
|
||||
Body io.Reader
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
}
|
||||
|
||||
type Content struct {
|
||||
Attachment model.Attachment
|
||||
Variant model.Variant
|
||||
Body io.ReadCloser
|
||||
}
|
||||
|
||||
@@ -69,7 +73,7 @@ type Service struct {
|
||||
}
|
||||
|
||||
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 {
|
||||
return nil, errors.New("invalid attachment service configuration")
|
||||
}
|
||||
if config.BodyMemoryBytes < config.MaxBytes {
|
||||
@@ -96,58 +100,38 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
|
||||
defer release()
|
||||
|
||||
attachmentID := uuid.NewString()
|
||||
objectKey := "attachments/" + uuid.NewString() + "/original"
|
||||
stagingKey := "staging/" + attachmentID + "/" + uuid.NewString()
|
||||
digest := sha256.Sum256(data)
|
||||
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
|
||||
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
|
||||
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
||||
ObjectKey: objectKey, Status: model.AttachmentPending,
|
||||
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending,
|
||||
PipelineVersion: s.config.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
|
||||
s.deletePending(pending.ID, objectKey)
|
||||
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
||||
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
|
||||
s.deletePending(pending.ID, stagingKey)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
|
||||
if err == nil {
|
||||
return attachment, nil
|
||||
createdAt := time.Now().UTC()
|
||||
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
|
||||
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
|
||||
}, input.Traceparent, input.Tracestate)
|
||||
if err != nil {
|
||||
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
return s.resolveReadyOutcome(pending.ID, objectKey)
|
||||
return queued, nil
|
||||
}
|
||||
|
||||
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
|
||||
func (s *Service) deletePending(attachmentID, stagingKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
|
||||
if err == nil && stored.Status == model.AttachmentReady {
|
||||
return stored, nil
|
||||
}
|
||||
if err == nil && stored.Status == model.AttachmentPending {
|
||||
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
|
||||
}
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||
}
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
|
||||
func (s *Service) deletePending(attachmentID, objectKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -160,16 +144,16 @@ func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (mo
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
|
||||
attachment, err := s.Metadata(ctx, attachmentID, userID)
|
||||
func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) {
|
||||
attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName)
|
||||
if err != nil {
|
||||
return Content{}, err
|
||||
return Content{}, mapDependencyError(err)
|
||||
}
|
||||
body, err := s.objects.Get(ctx, attachment.ObjectKey)
|
||||
body, err := s.objects.Get(ctx, variant.ObjectKey)
|
||||
if err != nil {
|
||||
return Content{}, model.ErrUnavailable
|
||||
}
|
||||
return Content{Attachment: attachment, Body: body}, nil
|
||||
return Content{Attachment: attachment, Variant: variant, Body: body}, nil
|
||||
}
|
||||
|
||||
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
|
||||
|
||||
@@ -27,8 +27,8 @@ func TestUploadLifecycle(t *testing.T) {
|
||||
|
||||
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, model.AttachmentReady, result.Status)
|
||||
require.Equal(t, []string{"pending", "put", "ready"}, events)
|
||||
require.Equal(t, model.AttachmentQueued, result.Status)
|
||||
require.Equal(t, []string{"pending", "put", "queued"}, events)
|
||||
}
|
||||
|
||||
func TestUploadValidation(t *testing.T) {
|
||||
@@ -86,7 +86,7 @@ func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
|
||||
Config{
|
||||
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
|
||||
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
|
||||
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1,
|
||||
},
|
||||
)
|
||||
require.Error(t, err)
|
||||
@@ -103,16 +103,16 @@ func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
|
||||
require.True(t, repository.pendingDeleted)
|
||||
}
|
||||
|
||||
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
|
||||
func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
|
||||
repository := &repositoryStub{queueErr: errors.New("connection lost")}
|
||||
objects := &objectStub{}
|
||||
service := newTestService(t, repository, objects, 1024, 4)
|
||||
|
||||
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, model.AttachmentReady, result.Status)
|
||||
_, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrUnavailable)
|
||||
require.False(t, objects.deleted)
|
||||
require.False(t, repository.pendingDeleted)
|
||||
}
|
||||
|
||||
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
|
||||
@@ -121,7 +121,7 @@ func newTestService(t *testing.T, repository Repository, objects ObjectStore, ma
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
|
||||
Config{
|
||||
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
|
||||
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
|
||||
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1,
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
@@ -137,8 +137,7 @@ func upload(data []byte, contentType string) Upload {
|
||||
|
||||
type repositoryStub struct {
|
||||
events *[]string
|
||||
markErr error
|
||||
lookupReady bool
|
||||
queueErr error
|
||||
pendingDeleted bool
|
||||
pending model.Attachment
|
||||
}
|
||||
@@ -154,12 +153,19 @@ func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment m
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.markErr != nil {
|
||||
return model.Attachment{}, s.markErr
|
||||
func (s *repositoryStub) QueueAttachmentWithJobAndOutbox(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
model.AttachmentJob,
|
||||
string,
|
||||
string,
|
||||
) (model.Attachment, error) {
|
||||
if s.queueErr != nil {
|
||||
return model.Attachment{}, s.queueErr
|
||||
}
|
||||
s.pending.Status = model.AttachmentReady
|
||||
s.record("ready")
|
||||
s.pending.Status = model.AttachmentQueued
|
||||
s.record("queued")
|
||||
return s.pending, nil
|
||||
}
|
||||
|
||||
@@ -168,18 +174,14 @@ func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.lookupReady {
|
||||
s.pending.Status = model.AttachmentReady
|
||||
return s.pending, nil
|
||||
}
|
||||
return model.Attachment{}, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.pending, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) {
|
||||
return s.pending, model.Variant{}, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) record(event string) {
|
||||
if s.events != nil {
|
||||
*s.events = append(*s.events, event)
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
)
|
||||
|
||||
var errInvalidEXIF = errors.New("invalid EXIF metadata")
|
||||
|
||||
func jpegOrientation(ctx context.Context, data []byte) (int, error) {
|
||||
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for offset := 2; offset < len(data); {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if data[offset] != 0xff {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for offset < len(data) && data[offset] == 0xff {
|
||||
offset++
|
||||
}
|
||||
if offset >= len(data) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
marker := data[offset]
|
||||
offset++
|
||||
if marker == 0xd9 || marker == 0xda {
|
||||
return 1, nil
|
||||
}
|
||||
if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 {
|
||||
continue
|
||||
}
|
||||
if offset+2 > len(data) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
length := int(binary.BigEndian.Uint16(data[offset : offset+2]))
|
||||
if length < 2 || offset+length > len(data) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
payload := data[offset+2 : offset+length]
|
||||
offset += length
|
||||
if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" {
|
||||
continue
|
||||
}
|
||||
return tiffOrientation(ctx, payload[6:])
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func tiffOrientation(ctx context.Context, data []byte) (int, error) {
|
||||
if len(data) < 8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
var order binary.ByteOrder
|
||||
switch string(data[:2]) {
|
||||
case "II":
|
||||
order = binary.LittleEndian
|
||||
case "MM":
|
||||
order = binary.BigEndian
|
||||
default:
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
if order.Uint16(data[2:4]) != 42 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
ifdOffset := uint64(order.Uint32(data[4:8]))
|
||||
if ifdOffset > uint64(len(data)-2) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2]))
|
||||
entriesStart := ifdOffset + 2
|
||||
if count > (uint64(len(data))-entriesStart)/12 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for index := uint64(0); index < count; index++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
entry := data[entriesStart+index*12 : entriesStart+(index+1)*12]
|
||||
if order.Uint16(entry[:2]) != 0x0112 {
|
||||
continue
|
||||
}
|
||||
if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
orientation := int(order.Uint16(entry[8:10]))
|
||||
if orientation < 1 || orientation > 8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
return orientation, nil
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
width, height := bounds.Dx(), bounds.Dy()
|
||||
outputWidth, outputHeight := width, height
|
||||
if orientation >= 5 {
|
||||
outputWidth, outputHeight = height, width
|
||||
}
|
||||
output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight))
|
||||
for y := 0; y < outputHeight; y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for x := 0; x < outputWidth; x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation)
|
||||
output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY))
|
||||
}
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
|
||||
switch orientation {
|
||||
case 2:
|
||||
return width - 1 - x, y
|
||||
case 3:
|
||||
return width - 1 - x, height - 1 - y
|
||||
case 4:
|
||||
return x, height - 1 - y
|
||||
case 5:
|
||||
return y, x
|
||||
case 6:
|
||||
return y, height - 1 - x
|
||||
case 7:
|
||||
return width - 1 - y, height - 1 - x
|
||||
case 8:
|
||||
return width - 1 - y, x
|
||||
default:
|
||||
return x, y
|
||||
}
|
||||
}
|
||||
|
||||
func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := 0; y < bounds.Dy(); y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for x := 0; x < bounds.Dx(); x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA()
|
||||
output.SetNRGBA(x, y, color.NRGBA{
|
||||
R: uint8((r + 0xffff - a) / 0x101),
|
||||
G: uint8((g + 0xffff - a) / 0x101),
|
||||
B: uint8((b + 0xffff - a) / 0x101),
|
||||
A: 0xff,
|
||||
})
|
||||
}
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
"github.com/deepteams/webp"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultJPEGQuality is used for thumbnail and crop variants.
|
||||
DefaultJPEGQuality = 85
|
||||
// DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size.
|
||||
DefaultWebPQuality = 82
|
||||
// DefaultWebPMethod selects the encoder's balanced quality and CPU mode.
|
||||
DefaultWebPMethod = 4
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidConfig = errors.New("invalid image transform configuration")
|
||||
ErrUnsupportedMedia = errors.New("unsupported image media type")
|
||||
ErrPayloadTooLarge = errors.New("image exceeds configured limits")
|
||||
ErrInvalidImage = errors.New("invalid image data")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
JPEGQuality int
|
||||
WebPQuality float32
|
||||
WebPMethod int
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Bytes []byte
|
||||
ContentType string
|
||||
Width int
|
||||
Height int
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Original Variant
|
||||
Thumbnail Variant
|
||||
Crop Variant
|
||||
Optimized Variant
|
||||
Orientation int
|
||||
}
|
||||
|
||||
type Processor struct {
|
||||
config Config
|
||||
}
|
||||
|
||||
func DefaultConfig(maxBytes, maxPixels int64) Config {
|
||||
return Config{
|
||||
MaxBytes: maxBytes, MaxPixels: maxPixels,
|
||||
JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod,
|
||||
}
|
||||
}
|
||||
|
||||
func NewProcessor(config Config) (*Processor, error) {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 ||
|
||||
config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
return &Processor{config: config}, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
oriented, err := applyOrientation(ctx, img, orientation)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
opaque, err := flattenWhite(ctx, oriented)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result := Result{
|
||||
Original: variant(data, format, width, height),
|
||||
Orientation: orientation,
|
||||
}
|
||||
|
||||
thumbnail, err := fitContext(ctx, opaque, 256, 256, false)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
crop := centeredSquare(opaque)
|
||||
crop, err = resizeContext(ctx, crop, 128, 128)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
optimized, err := fitContext(ctx, opaque, 1280, 1280, false)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *Processor) decode(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
declaredContentType string,
|
||||
) (image.Image, string, int, int, int, error) {
|
||||
if declaredContentType != "image/jpeg" && declaredContentType != "image/png" {
|
||||
return nil, "", 0, 0, 0, ErrUnsupportedMedia
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
if int64(len(data)) > p.config.MaxBytes {
|
||||
return nil, "", 0, 0, 0, ErrPayloadTooLarge
|
||||
}
|
||||
if http.DetectContentType(data) != declaredContentType {
|
||||
return nil, "", 0, 0, 0, ErrUnsupportedMedia
|
||||
}
|
||||
config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
||||
if contextErr := contextError(ctx); contextErr != nil {
|
||||
return nil, "", 0, 0, 0, contextErr
|
||||
}
|
||||
if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
if int64(config.Width) > p.config.MaxPixels/int64(config.Height) {
|
||||
return nil, "", 0, 0, 0, ErrPayloadTooLarge
|
||||
}
|
||||
img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
||||
if contextErr := contextError(ctx); contextErr != nil {
|
||||
return nil, "", 0, 0, 0, contextErr
|
||||
}
|
||||
if err != nil || decodedFormat != format {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
orientation := 1
|
||||
if format == "jpeg" {
|
||||
orientation, err = jpegOrientation(ctx, data)
|
||||
if err != nil {
|
||||
if contextErr := contextError(ctx); contextErr != nil {
|
||||
return nil, "", 0, 0, 0, contextErr
|
||||
}
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
}
|
||||
return img, declaredContentType, config.Width, config.Height, orientation, nil
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r contextReader) Read(buffer []byte) (int, error) {
|
||||
if err := contextError(r.ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(buffer)
|
||||
}
|
||||
|
||||
func contextError(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func contentType(format string) string {
|
||||
switch format {
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func encodeJPEG(img image.Image, quality int) (Variant, error) {
|
||||
var output bytes.Buffer
|
||||
if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil {
|
||||
return Variant{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil
|
||||
}
|
||||
|
||||
func encodeWebP(img image.Image, quality float32, method int) (Variant, error) {
|
||||
var output bytes.Buffer
|
||||
options := &webp.EncoderOptions{Quality: quality, Method: method}
|
||||
if err := webp.Encode(&output, img, options); err != nil {
|
||||
return Variant{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil
|
||||
}
|
||||
|
||||
func variant(data []byte, contentType string, width, height int) Variant {
|
||||
digest := sha256.Sum256(data)
|
||||
return Variant{
|
||||
Bytes: data, ContentType: contentType, Width: width, Height: height,
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
}
|
||||
}
|
||||
|
||||
// EstimateMemoryWeight returns a conservative reservation for the input, decoded source,
|
||||
// orientation and alpha copies, variant working buffer, and encoded variant buffers.
|
||||
func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) {
|
||||
if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) {
|
||||
return 0, ErrInvalidImage
|
||||
}
|
||||
pixels := int64(width) * int64(height)
|
||||
optimizedPixels := fittedPixels(width, height, 1280, 1280)
|
||||
thumbnailPixels := fittedPixels(width, height, 256, 256)
|
||||
const cropPixels int64 = 128 * 128
|
||||
if pixels > (math.MaxInt64-inputBytes)/16 {
|
||||
return 0, ErrPayloadTooLarge
|
||||
}
|
||||
weight := inputBytes + pixels*16
|
||||
variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4
|
||||
if variants > math.MaxInt64-weight {
|
||||
return 0, ErrPayloadTooLarge
|
||||
}
|
||||
return weight + variants, nil
|
||||
}
|
||||
|
||||
func fittedPixels(width, height, maxWidth, maxHeight int) int64 {
|
||||
newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false)
|
||||
return int64(newWidth) * int64(newHeight)
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"github.com/deepteams/webp"
|
||||
)
|
||||
|
||||
func TestProcessProducesExpectedVariants(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 320, 200))
|
||||
fillGradient(source)
|
||||
data := encodePNG(t, source)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 320*200)
|
||||
|
||||
result, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("process image: %v", err)
|
||||
}
|
||||
|
||||
assertVariant(t, result.Original, "image/png", 320, 200)
|
||||
assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160)
|
||||
assertVariant(t, result.Crop, "image/jpeg", 128, 128)
|
||||
assertVariant(t, result.Optimized, "image/webp", 320, 200)
|
||||
if !bytes.Equal(result.Original.Bytes, data) {
|
||||
t.Fatal("original bytes changed")
|
||||
}
|
||||
digest := sha256.Sum256(data)
|
||||
if result.Original.SHA256 != hex.EncodeToString(digest[:]) {
|
||||
t.Fatalf("unexpected original digest: %s", result.Original.SHA256)
|
||||
}
|
||||
if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil {
|
||||
t.Fatalf("decode optimized WebP: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessDoesNotUpscaleFitVariants(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 64, 32))
|
||||
fillGradient(source)
|
||||
data := encodePNG(t, source)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 64*32)
|
||||
|
||||
result, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("process image: %v", err)
|
||||
}
|
||||
|
||||
assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32)
|
||||
assertVariant(t, result.Optimized, "image/webp", 64, 32)
|
||||
assertVariant(t, result.Crop, "image/jpeg", 128, 128)
|
||||
}
|
||||
|
||||
func TestFlattenWhiteCompositesAlpha(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 2, 1))
|
||||
source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0})
|
||||
source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128})
|
||||
|
||||
result, err := flattenWhite(context.Background(), source)
|
||||
if err != nil {
|
||||
t.Fatalf("flatten image: %v", err)
|
||||
}
|
||||
if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) {
|
||||
t.Fatalf("transparent pixel: got %#v", got)
|
||||
}
|
||||
if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) {
|
||||
t.Fatalf("semi-transparent pixel: got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessAppliesEXIFOrientation(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 3, 2))
|
||||
fillGradient(source)
|
||||
data := injectOrientation(t, encodeJPEGForTest(t, source), 6)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 3*2)
|
||||
|
||||
result, err := processor.Process(context.Background(), data, "image/jpeg")
|
||||
if err != nil {
|
||||
t.Fatalf("process image: %v", err)
|
||||
}
|
||||
|
||||
if result.Orientation != 6 {
|
||||
t.Fatalf("orientation: got %d, want 6", result.Orientation)
|
||||
}
|
||||
assertVariant(t, result.Original, "image/jpeg", 3, 2)
|
||||
assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3)
|
||||
assertVariant(t, result.Optimized, "image/webp", 2, 3)
|
||||
}
|
||||
|
||||
func TestProcessRejectsInvalidInputs(t *testing.T) {
|
||||
valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10)))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
contentType string
|
||||
maxPixels int64
|
||||
want error
|
||||
}{
|
||||
{name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia},
|
||||
{name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia},
|
||||
{name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge},
|
||||
{name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels)
|
||||
_, err := processor.Process(context.Background(), test.data, test.contentType)
|
||||
if !errors.Is(err, test.want) {
|
||||
t.Fatalf("error: got %v, want %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIsDeterministic(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 48, 31))
|
||||
fillGradient(source)
|
||||
data := encodePNG(t, source)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 48*31)
|
||||
|
||||
first, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("first process: %v", err)
|
||||
}
|
||||
second, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("second process: %v", err)
|
||||
}
|
||||
|
||||
for name, variants := range map[string][2]Variant{
|
||||
"thumbnail": {first.Thumbnail, second.Thumbnail},
|
||||
"crop": {first.Crop, second.Crop},
|
||||
"optimized": {first.Optimized, second.Optimized},
|
||||
} {
|
||||
if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) {
|
||||
t.Fatalf("%s bytes differ", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessHonorsCanceledContext(t *testing.T) {
|
||||
processor := newTestProcessor(t, 1024, 100)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := processor.Process(ctx, []byte("unused"), "image/png")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error: got %v, want %v", err, context.Canceled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeStopsAfterCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
source := cancelingImage{
|
||||
Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)),
|
||||
cancel: cancel,
|
||||
}
|
||||
_, err := resizeContext(ctx, source, 128, 128)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error: got %v, want %v", err, context.Canceled)
|
||||
}
|
||||
}
|
||||
|
||||
type cancelingImage struct {
|
||||
image.Image
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (i cancelingImage) At(x, y int) color.Color {
|
||||
i.cancel()
|
||||
return i.Image.At(x, y)
|
||||
}
|
||||
|
||||
func TestEstimateMemoryWeight(t *testing.T) {
|
||||
weight, err := EstimateMemoryWeight(1000, 320, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("estimate memory: %v", err)
|
||||
}
|
||||
want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4)
|
||||
if weight != want {
|
||||
t.Fatalf("weight: got %d, want %d", weight, want)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor {
|
||||
t.Helper()
|
||||
processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels))
|
||||
if err != nil {
|
||||
t.Fatalf("new processor: %v", err)
|
||||
}
|
||||
return processor
|
||||
}
|
||||
|
||||
func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) {
|
||||
t.Helper()
|
||||
if variant.ContentType != contentType || variant.Width != width || variant.Height != height {
|
||||
t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d",
|
||||
variant.ContentType, variant.Width, variant.Height, contentType, width, height)
|
||||
}
|
||||
if len(variant.Bytes) == 0 || variant.SHA256 == "" {
|
||||
t.Fatal("variant has no bytes or digest")
|
||||
}
|
||||
}
|
||||
|
||||
func fillGradient(img *image.NRGBA) {
|
||||
for y := 0; y < img.Bounds().Dy(); y++ {
|
||||
for x := 0; x < img.Bounds().Dx(); x++ {
|
||||
img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func encodePNG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
if err := png.Encode(&output, img); err != nil {
|
||||
t.Fatalf("encode PNG: %v", err)
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
func encodeJPEGForTest(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil {
|
||||
t.Fatalf("encode JPEG: %v", err)
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte {
|
||||
t.Helper()
|
||||
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
|
||||
t.Fatal("not a JPEG")
|
||||
}
|
||||
tiff := make([]byte, 26)
|
||||
copy(tiff[:2], "II")
|
||||
binary.LittleEndian.PutUint16(tiff[2:4], 42)
|
||||
binary.LittleEndian.PutUint32(tiff[4:8], 8)
|
||||
binary.LittleEndian.PutUint16(tiff[8:10], 1)
|
||||
binary.LittleEndian.PutUint16(tiff[10:12], 0x0112)
|
||||
binary.LittleEndian.PutUint16(tiff[12:14], 3)
|
||||
binary.LittleEndian.PutUint32(tiff[14:18], 1)
|
||||
binary.LittleEndian.PutUint16(tiff[18:20], orientation)
|
||||
payload := append([]byte("Exif\x00\x00"), tiff...)
|
||||
segment := []byte{0xff, 0xe1, 0, 0}
|
||||
binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2))
|
||||
result := make([]byte, 0, len(data)+len(segment)+len(payload))
|
||||
result = append(result, data[:2]...)
|
||||
result = append(result, segment...)
|
||||
result = append(result, payload...)
|
||||
return append(result, data[2:]...)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"image"
|
||||
"image/color"
|
||||
)
|
||||
|
||||
func fitContext(
|
||||
ctx context.Context,
|
||||
source image.Image,
|
||||
maxWidth int,
|
||||
maxHeight int,
|
||||
upscale bool,
|
||||
) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale)
|
||||
return resizeContext(ctx, source, width, height)
|
||||
}
|
||||
|
||||
func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) {
|
||||
if !upscale && width <= maxWidth && height <= maxHeight {
|
||||
return width, height
|
||||
}
|
||||
if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) {
|
||||
newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width)))
|
||||
return maxWidth, newHeight
|
||||
}
|
||||
newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height)))
|
||||
return newWidth, maxHeight
|
||||
}
|
||||
|
||||
func centeredSquare(source image.Image) image.Image {
|
||||
bounds := source.Bounds()
|
||||
size := min(bounds.Dx(), bounds.Dy())
|
||||
left := bounds.Min.X + (bounds.Dx()-size)/2
|
||||
top := bounds.Min.Y + (bounds.Dy()-size)/2
|
||||
return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)}
|
||||
}
|
||||
|
||||
type croppedImage struct {
|
||||
source image.Image
|
||||
bounds image.Rectangle
|
||||
offset image.Point
|
||||
}
|
||||
|
||||
func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() }
|
||||
func (i croppedImage) Bounds() image.Rectangle { return i.bounds }
|
||||
func (i croppedImage) At(x, y int) color.Color {
|
||||
return i.source.At(i.offset.X+x, i.offset.Y+y)
|
||||
}
|
||||
|
||||
func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
if bounds.Dx() == width && bounds.Dy() == height {
|
||||
return copyNRGBAContext(ctx, source)
|
||||
}
|
||||
output := image.NewNRGBA(image.Rect(0, 0, width, height))
|
||||
for y := 0; y < height; y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
y0, y1, fy := sampleCoordinate(y, height, bounds.Dy())
|
||||
for x := 0; x < width; x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
x0, x1, fx := sampleCoordinate(x, width, bounds.Dx())
|
||||
c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA)
|
||||
c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA)
|
||||
c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA)
|
||||
c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA)
|
||||
output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy))
|
||||
}
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) {
|
||||
coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15
|
||||
if coordinate <= 0 {
|
||||
return 0, min(1, sourceSize-1), 0
|
||||
}
|
||||
maximum := int64(sourceSize-1) << 16
|
||||
if coordinate >= maximum {
|
||||
return sourceSize - 1, sourceSize - 1, 0
|
||||
}
|
||||
first := int(coordinate >> 16)
|
||||
return first, first + 1, uint32(coordinate & 0xffff)
|
||||
}
|
||||
|
||||
func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA {
|
||||
return color.NRGBA{
|
||||
R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy),
|
||||
G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy),
|
||||
B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy),
|
||||
A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy),
|
||||
}
|
||||
}
|
||||
|
||||
func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 {
|
||||
const one uint64 = 1 << 16
|
||||
top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx)
|
||||
bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx)
|
||||
value := top*(one-uint64(fy)) + bottom*uint64(fy)
|
||||
return uint8((value + 1<<31) >> 32)
|
||||
}
|
||||
|
||||
func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := 0; y < bounds.Dy(); y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for x := 0; x < bounds.Dx(); x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y))
|
||||
}
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
|
||||
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
|
||||
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
||||
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
|
||||
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
|
||||
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
|
||||
}
|
||||
|
||||
type Broker interface {
|
||||
Read(context.Context) ([]redisstream.Delivery, error)
|
||||
Reclaim(context.Context, string) ([]redisstream.Delivery, string, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
Get(context.Context, string) (io.ReadCloser, error)
|
||||
Put(context.Context, string, string, []byte) error
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
type ImageProcessor interface {
|
||||
Process(context.Context, []byte, string) (transform.Result, error)
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObserveStarted()
|
||||
ObserveFinished(string, time.Duration)
|
||||
ObserveRetry()
|
||||
ObserveReclaimed(int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
WorkerID string
|
||||
Concurrency int
|
||||
LeaseDuration time.Duration
|
||||
HeartbeatInterval time.Duration
|
||||
ProcessingTimeout time.Duration
|
||||
ReclaimInterval time.Duration
|
||||
MaxAttempts int
|
||||
RetryBase time.Duration
|
||||
RetryMax time.Duration
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
MemoryBytes int64
|
||||
}
|
||||
|
||||
type FaultPoint string
|
||||
|
||||
const (
|
||||
FaultNone FaultPoint = ""
|
||||
FaultBeforeSideEffect FaultPoint = "before_side_effect"
|
||||
FaultAfterSideEffect FaultPoint = "after_side_effect"
|
||||
FaultAfterCommit FaultPoint = "after_commit"
|
||||
)
|
||||
|
||||
var ErrInjectedFailure = errors.New("injected worker failure")
|
||||
|
||||
type Worker struct {
|
||||
logger *slog.Logger
|
||||
repository Repository
|
||||
broker Broker
|
||||
objects ObjectStore
|
||||
processor ImageProcessor
|
||||
metrics Metrics
|
||||
config Config
|
||||
memory *semaphore.Weighted
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(
|
||||
logger *slog.Logger,
|
||||
repository Repository,
|
||||
broker Broker,
|
||||
objects ObjectStore,
|
||||
processor ImageProcessor,
|
||||
metrics Metrics,
|
||||
config Config,
|
||||
) (*Worker, error) {
|
||||
if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil {
|
||||
return nil, errors.New("worker dependencies are required")
|
||||
}
|
||||
if err := validateConfig(config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = nopMetrics{}
|
||||
}
|
||||
return &Worker{
|
||||
logger: logger, repository: repository, broker: broker, objects: objects,
|
||||
processor: processor, metrics: metrics, config: config,
|
||||
memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateConfig(config Config) error {
|
||||
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
|
||||
config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration ||
|
||||
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
|
||||
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
|
||||
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
|
||||
return errors.New("invalid worker configuration")
|
||||
}
|
||||
const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128
|
||||
if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 ||
|
||||
config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes {
|
||||
return errors.New("worker memory must fit one maximum image")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) error {
|
||||
lastReclaim := time.Time{}
|
||||
reclaimCursor := "0-0"
|
||||
for ctx.Err() == nil {
|
||||
deliveries, err := w.broker.Read(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
w.processBatch(ctx, deliveries)
|
||||
if w.now().Sub(lastReclaim) < w.config.ReclaimInterval {
|
||||
continue
|
||||
}
|
||||
recovery, err := w.repository.RecoverExpiredAttachmentJobs(
|
||||
ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recover expired attachment jobs: %w", err)
|
||||
}
|
||||
if recovery.Requeued > 0 || recovery.Failed > 0 {
|
||||
w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed)
|
||||
}
|
||||
deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
w.metrics.ObserveReclaimed(len(deliveries))
|
||||
w.processBatch(ctx, deliveries)
|
||||
reclaimCursor = next
|
||||
if reclaimCursor == "" {
|
||||
reclaimCursor = "0-0"
|
||||
}
|
||||
lastReclaim = w.now()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) {
|
||||
limit := make(chan struct{}, w.config.Concurrency)
|
||||
var group sync.WaitGroup
|
||||
for _, delivery := range deliveries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
limit <- struct{}{}
|
||||
group.Add(1)
|
||||
go func(delivery redisstream.Delivery) {
|
||||
defer group.Done()
|
||||
defer func() { <-limit }()
|
||||
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
|
||||
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
|
||||
}
|
||||
}(delivery)
|
||||
}
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
|
||||
started := w.now()
|
||||
w.metrics.ObserveStarted()
|
||||
result := "transient_error"
|
||||
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
|
||||
|
||||
if delivery.DecodeError != nil {
|
||||
result = "invalid_job"
|
||||
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
||||
}
|
||||
envelope := delivery.Envelope
|
||||
lease, err := w.repository.ClaimAttachmentJob(
|
||||
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
|
||||
w.config.WorkerID, w.config.LeaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
result, err = w.resolveClaim(ctx, delivery, envelope, err)
|
||||
return err
|
||||
}
|
||||
if fault == FaultBeforeSideEffect {
|
||||
result = "injected_failure"
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
|
||||
processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout)
|
||||
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
|
||||
heartbeatDone := make(chan error, 1)
|
||||
go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }()
|
||||
variants, err := w.process(processingCtx, lease)
|
||||
cancelProcessing()
|
||||
cancelHeartbeat()
|
||||
heartbeatErr := <-heartbeatDone
|
||||
if heartbeatErr != nil {
|
||||
return heartbeatErr
|
||||
}
|
||||
if err != nil {
|
||||
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
|
||||
return err
|
||||
}
|
||||
if fault == FaultAfterSideEffect {
|
||||
result = "injected_failure"
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete attachment job: %w", err)
|
||||
}
|
||||
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
|
||||
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
|
||||
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
||||
}
|
||||
}
|
||||
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
|
||||
return fmt.Errorf("unexpected completion result %q", mutation)
|
||||
}
|
||||
if fault == FaultAfterCommit {
|
||||
result = "injected_failure"
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
if err := delivery.Ack(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
result = "ready"
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) maintainLease(
|
||||
ctx context.Context,
|
||||
cancelProcessing context.CancelFunc,
|
||||
lease model.AttachmentJobLease,
|
||||
) error {
|
||||
ticker := time.NewTicker(w.config.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
cancelProcessing()
|
||||
return fmt.Errorf("renew attachment job lease: %w", err)
|
||||
}
|
||||
if !renewed {
|
||||
cancelProcessing()
|
||||
return model.ErrStaleLease
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) resolveClaim(
|
||||
ctx context.Context,
|
||||
delivery redisstream.Delivery,
|
||||
envelope redisstream.Envelope,
|
||||
claimErr error,
|
||||
) (string, error) {
|
||||
switch {
|
||||
case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput):
|
||||
return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
||||
case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline):
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
case errors.Is(claimErr, model.ErrJobBusy):
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
case errors.Is(claimErr, model.ErrJobNotDue):
|
||||
return "not_due", nil
|
||||
case errors.Is(claimErr, model.ErrConflict):
|
||||
job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String())
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("inspect conflicting job: %w", err)
|
||||
}
|
||||
switch job.Status {
|
||||
case model.AttachmentJobReady:
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
case model.AttachmentJobFailed:
|
||||
code := "processing_failed"
|
||||
if job.ErrorCode != nil {
|
||||
code = *job.ErrorCode
|
||||
}
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
default:
|
||||
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
||||
}
|
||||
default:
|
||||
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) {
|
||||
body, err := w.objects.Get(ctx, lease.Attachment.StagingKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > w.config.MaxBytes {
|
||||
return nil, transform.ErrPayloadTooLarge
|
||||
}
|
||||
imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, transform.ErrInvalidImage
|
||||
}
|
||||
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if weight > w.config.MemoryBytes {
|
||||
return nil, errWorkerCapacity
|
||||
}
|
||||
if err := w.memory.Acquire(ctx, weight); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer w.memory.Release(weight)
|
||||
|
||||
processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
variants := []struct {
|
||||
name string
|
||||
value transform.Variant
|
||||
}{
|
||||
{name: "original", value: processed.Original},
|
||||
{name: "thumbnail", value: processed.Thumbnail},
|
||||
{name: "crop", value: processed.Crop},
|
||||
{name: "optimized", value: processed.Optimized},
|
||||
}
|
||||
result := make([]model.Variant, 0, len(variants))
|
||||
for _, item := range variants {
|
||||
objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value)
|
||||
if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, model.Variant{
|
||||
Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)),
|
||||
SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height,
|
||||
ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r contextReader) Read(buffer []byte) (int, error) {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return 0, r.ctx.Err()
|
||||
default:
|
||||
return r.reader.Read(buffer)
|
||||
}
|
||||
}
|
||||
|
||||
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
|
||||
|
||||
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
|
||||
extension := ".bin"
|
||||
switch value.ContentType {
|
||||
case "image/png":
|
||||
extension = ".png"
|
||||
case "image/jpeg":
|
||||
extension = ".jpg"
|
||||
case "image/webp":
|
||||
extension = ".webp"
|
||||
}
|
||||
return path.Join(
|
||||
"attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion),
|
||||
name, value.SHA256+extension,
|
||||
)
|
||||
}
|
||||
|
||||
func (w *Worker) resolveProcessingFailure(
|
||||
ctx context.Context,
|
||||
delivery redisstream.Delivery,
|
||||
lease model.AttachmentJobLease,
|
||||
processingErr error,
|
||||
) (string, error) {
|
||||
code, permanent := processingError(processingErr)
|
||||
if permanent || lease.Job.Attempt >= w.config.MaxAttempts {
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
mutation, err := w.repository.FailAttachmentJob(ctx, lease, code)
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("fail attachment job: %w", err)
|
||||
}
|
||||
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
}
|
||||
if mutation != model.JobMutationApplied {
|
||||
return "transient_error", fmt.Errorf("unexpected failure result %q", mutation)
|
||||
}
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
}
|
||||
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
|
||||
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
|
||||
}
|
||||
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
}
|
||||
if mutation != model.JobMutationApplied {
|
||||
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
|
||||
}
|
||||
w.metrics.ObserveRetry()
|
||||
return "retry", delivery.Ack(ctx)
|
||||
}
|
||||
|
||||
func processingError(err error) (string, bool) {
|
||||
switch {
|
||||
case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia):
|
||||
return "invalid_image", true
|
||||
case errors.Is(err, transform.ErrPayloadTooLarge):
|
||||
return "image_too_large", true
|
||||
case errors.Is(err, objectstore.ErrNotFound):
|
||||
return "source_missing", true
|
||||
case errors.Is(err, errWorkerCapacity):
|
||||
return "worker_capacity", true
|
||||
default:
|
||||
return "processing_failed", false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
|
||||
delay := w.config.RetryBase
|
||||
for step := 1; step < attempt && delay < w.config.RetryMax; step++ {
|
||||
if delay > w.config.RetryMax/2 {
|
||||
delay = w.config.RetryMax
|
||||
break
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
if delay > w.config.RetryMax {
|
||||
delay = w.config.RetryMax
|
||||
}
|
||||
var hash uint64
|
||||
for index := range jobID {
|
||||
hash = hash*131 + uint64(jobID[index])
|
||||
}
|
||||
jitterRange := min(delay/4, w.config.RetryMax-delay)
|
||||
if jitterRange <= 0 {
|
||||
return delay
|
||||
}
|
||||
jitter := time.Duration(hash % uint64(jitterRange))
|
||||
if delay > time.Duration(math.MaxInt64)-jitter {
|
||||
return delay
|
||||
}
|
||||
return delay + jitter
|
||||
}
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObserveStarted() {}
|
||||
func (nopMetrics) ObserveFinished(string, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry() {}
|
||||
func (nopMetrics) ObserveReclaimed(int) {}
|
||||
@@ -0,0 +1,109 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
)
|
||||
|
||||
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
|
||||
config := validConfig()
|
||||
require.NoError(t, validateConfig(config))
|
||||
|
||||
config.MemoryBytes = 384 * 1024 * 1024
|
||||
require.ErrorContains(t, validateConfig(config), "memory must fit")
|
||||
}
|
||||
|
||||
func TestRetryDelayIsBounded(t *testing.T) {
|
||||
worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}}
|
||||
for attempt := 1; attempt <= 10; attempt++ {
|
||||
delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt)
|
||||
require.GreaterOrEqual(t, delay, time.Second)
|
||||
require.LessOrEqual(t, delay, 4*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessingErrorClassification(t *testing.T) {
|
||||
tests := []struct {
|
||||
err error
|
||||
code string
|
||||
permanent bool
|
||||
}{
|
||||
{err: transform.ErrInvalidImage, code: "invalid_image", permanent: true},
|
||||
{err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true},
|
||||
{err: objectstore.ErrNotFound, code: "source_missing", permanent: true},
|
||||
{err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
code, permanent := processingError(test.err)
|
||||
require.Equal(t, test.code, code)
|
||||
require.Equal(t, test.permanent, permanent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) {
|
||||
var renewals atomic.Int32
|
||||
repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) {
|
||||
renewals.Add(1)
|
||||
return true, nil
|
||||
}}
|
||||
config := validConfig()
|
||||
config.HeartbeatInterval = 5 * time.Millisecond
|
||||
worker := &Worker{repository: repository, config: config}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }()
|
||||
require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond)
|
||||
cancel()
|
||||
require.NoError(t, <-done)
|
||||
}
|
||||
|
||||
func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) {
|
||||
repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) {
|
||||
return false, nil
|
||||
}}
|
||||
config := validConfig()
|
||||
config.HeartbeatInterval = time.Millisecond
|
||||
worker := &Worker{repository: repository, config: config}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
err := worker.maintainLease(ctx, cancel, testLease())
|
||||
require.ErrorIs(t, err, model.ErrStaleLease)
|
||||
require.ErrorIs(t, ctx.Err(), context.Canceled)
|
||||
}
|
||||
|
||||
type repositoryStub struct {
|
||||
Repository
|
||||
renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
func (s *repositoryStub) RenewAttachmentJobLease(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
return s.renew(ctx, lease, duration)
|
||||
}
|
||||
|
||||
func testLease() model.AttachmentJobLease {
|
||||
return model.AttachmentJobLease{Job: model.AttachmentJob{
|
||||
ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1",
|
||||
}}
|
||||
}
|
||||
|
||||
func validConfig() Config {
|
||||
return Config{
|
||||
WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second,
|
||||
HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second,
|
||||
ReclaimInterval: 5 * time.Second,
|
||||
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
|
||||
MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -46,6 +47,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
|
||||
ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
|
||||
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, model.ErrCapacity) {
|
||||
@@ -58,7 +60,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
}
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
|
||||
writeJSON(w, http.StatusCreated, uploaded)
|
||||
writeJSON(w, http.StatusAccepted, uploaded)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -108,20 +110,21 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
|
||||
return authErr
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || !originalVariant(query) {
|
||||
variant, valid := attachmentVariant(query)
|
||||
if err != nil || !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
attachmentID, apiErr := attachmentIDFromRequest(r)
|
||||
if apiErr != nil {
|
||||
return apiErr
|
||||
}
|
||||
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
|
||||
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant)
|
||||
if err != nil {
|
||||
return attachmentError(err)
|
||||
}
|
||||
defer func() { _ = content.Body.Close() }()
|
||||
w.Header().Set("Content-Type", content.Attachment.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
|
||||
w.Header().Set("Content-Type", content.Variant.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10))
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, content.Body); err != nil {
|
||||
@@ -130,11 +133,35 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
|
||||
return nil
|
||||
}
|
||||
|
||||
func originalVariant(query url.Values) bool {
|
||||
func attachmentVariant(query url.Values) (string, bool) {
|
||||
if len(query) == 0 {
|
||||
return true
|
||||
return "original", true
|
||||
}
|
||||
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
|
||||
if len(query) != 1 || len(query["variant"]) != 1 {
|
||||
return "", false
|
||||
}
|
||||
switch query["variant"][0] {
|
||||
case "original", "thumbnail", "crop", "optimized":
|
||||
return query["variant"][0], true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
|
||||
|
||||
func traceparent(value string) string {
|
||||
if traceparentPattern.MatchString(value) {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func tracestate(value string) string {
|
||||
if len(value) <= 512 {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
|
||||
@@ -158,6 +185,8 @@ func attachmentError(err error) *apiError {
|
||||
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
|
||||
case errors.Is(err, model.ErrCapacity):
|
||||
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
|
||||
case errors.Is(err, model.ErrConflict):
|
||||
return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"}
|
||||
default:
|
||||
return unavailable()
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ type MessageRepository interface {
|
||||
type AttachmentService interface {
|
||||
Upload(context.Context, attachment.Upload) (model.Attachment, error)
|
||||
Metadata(context.Context, string, string) (model.Attachment, error)
|
||||
Content(context.Context, string, string) (attachment.Content, error)
|
||||
Content(context.Context, string, string, string) (attachment.Content, error)
|
||||
}
|
||||
|
||||
type HealthChecker interface {
|
||||
|
||||
@@ -11,6 +11,11 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var (
|
||||
requestDurationBuckets = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30}
|
||||
uploadDurationBuckets = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60}
|
||||
)
|
||||
|
||||
type metrics struct {
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
@@ -36,7 +41,7 @@ func newMetrics() *metrics {
|
||||
Subsystem: "http",
|
||||
Name: "request_duration_seconds",
|
||||
Help: "HTTP request duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
Buckets: requestDurationBuckets,
|
||||
}, []string{"method", "route"}),
|
||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger",
|
||||
@@ -59,7 +64,7 @@ func newMetrics() *metrics {
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_duration_seconds",
|
||||
Help: "Attachment upload duration in seconds.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
Buckets: uploadDurationBuckets,
|
||||
}),
|
||||
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
|
||||
@@ -87,8 +87,12 @@ func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model
|
||||
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.uploaded, s.err
|
||||
}
|
||||
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
|
||||
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
|
||||
func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) {
|
||||
return attachment.Content{
|
||||
Attachment: s.uploaded,
|
||||
Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes},
|
||||
Body: io.NopCloser(strings.NewReader("content")),
|
||||
}, s.err
|
||||
}
|
||||
|
||||
type sessionStub struct {
|
||||
|
||||
@@ -15,4 +15,10 @@ var (
|
||||
ErrPayloadTooLarge = errors.New("payload too large")
|
||||
ErrCapacity = errors.New("capacity exhausted")
|
||||
ErrUnavailable = errors.New("dependency unavailable")
|
||||
ErrStaleLease = errors.New("stale lease")
|
||||
ErrStaleAttempt = errors.New("stale attempt")
|
||||
ErrStalePipeline = errors.New("stale pipeline")
|
||||
ErrJobBusy = errors.New("job is already leased")
|
||||
ErrJobComplete = errors.New("job is already complete")
|
||||
ErrJobNotDue = errors.New("job is not due")
|
||||
)
|
||||
|
||||
+94
-24
@@ -6,9 +6,21 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
AttachmentPending = "pending"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentDeleting = "deleting"
|
||||
AttachmentPending = "pending"
|
||||
AttachmentQueued = "queued"
|
||||
AttachmentProcessing = "processing"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentFailed = "failed"
|
||||
AttachmentDeleting = "deleting"
|
||||
|
||||
AttachmentJobQueued = "queued"
|
||||
AttachmentJobProcessing = "processing"
|
||||
AttachmentJobReady = "ready"
|
||||
AttachmentJobFailed = "failed"
|
||||
|
||||
JobMutationApplied JobMutationResult = "applied"
|
||||
JobMutationAlreadyComplete JobMutationResult = "already_complete"
|
||||
JobMutationStale JobMutationResult = "stale"
|
||||
)
|
||||
|
||||
const timestampLayout = "2006-01-02T15:04:05.000000Z"
|
||||
@@ -42,21 +54,23 @@ type Message struct {
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
ReadyAt *time.Time `json:"-"`
|
||||
Referenced bool `json:"-"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
StagingKey string `json:"-"`
|
||||
ReadyAt *time.Time `json:"-"`
|
||||
Referenced bool `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type StoredObject struct {
|
||||
@@ -66,12 +80,68 @@ type StoredObject struct {
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type AttachmentJob struct {
|
||||
ID string
|
||||
AttachmentID string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
Status string
|
||||
AvailableAt time.Time
|
||||
LeaseToken string
|
||||
LeaseOwner string
|
||||
LeaseUntil *time.Time
|
||||
ErrorCode *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type AttachmentJobMessage struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
type AttachmentJobLease struct {
|
||||
Job AttachmentJob
|
||||
Attachment Attachment
|
||||
}
|
||||
|
||||
type OutboxEvent struct {
|
||||
ID string
|
||||
Topic string
|
||||
PartitionKey string
|
||||
Payload json.RawMessage
|
||||
JobID string
|
||||
Attempt int
|
||||
AvailableAt time.Time
|
||||
PublishTries int
|
||||
LeaseToken string
|
||||
LeaseUntil *time.Time
|
||||
LastErrorCode *string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type JobMutationResult string
|
||||
|
||||
type RecoveryResult struct {
|
||||
Requeued int
|
||||
Failed int
|
||||
}
|
||||
|
||||
type Position struct {
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
var errLeaseLost = errors.New("outbox lease lost")
|
||||
|
||||
type Store interface {
|
||||
ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error)
|
||||
MarkOutboxPublished(context.Context, string, string) (bool, error)
|
||||
ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error)
|
||||
RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error)
|
||||
DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error)
|
||||
}
|
||||
|
||||
type Publisher interface {
|
||||
PublishConfirmed(context.Context, []byte) (string, error)
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObservePublished()
|
||||
ObservePublishFailure()
|
||||
ObserveRecovered(int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
BatchSize int
|
||||
LeaseDuration time.Duration
|
||||
PollInterval time.Duration
|
||||
RecoveryInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
Retention time.Duration
|
||||
CleanupInterval time.Duration
|
||||
CleanupBatchSize int
|
||||
}
|
||||
|
||||
type Dispatcher struct {
|
||||
store Store
|
||||
publisher Publisher
|
||||
metrics Metrics
|
||||
config Config
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) {
|
||||
if store == nil {
|
||||
return nil, errors.New("outbox store is required")
|
||||
}
|
||||
if publisher == nil {
|
||||
return nil, errors.New("outbox publisher is required")
|
||||
}
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = nopMetrics{}
|
||||
}
|
||||
return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case c.BatchSize <= 0:
|
||||
return errors.New("outbox batch size must be positive")
|
||||
case c.LeaseDuration <= 0:
|
||||
return errors.New("outbox lease duration must be positive")
|
||||
case c.PollInterval <= 0:
|
||||
return errors.New("outbox poll interval must be positive")
|
||||
case c.RecoveryInterval <= 0:
|
||||
return errors.New("outbox recovery interval must be positive")
|
||||
case c.RetryDelay <= 0:
|
||||
return errors.New("outbox retry delay must be positive")
|
||||
case c.Retention <= 0:
|
||||
return errors.New("outbox retention must be positive")
|
||||
case c.CleanupInterval <= 0:
|
||||
return errors.New("outbox cleanup interval must be positive")
|
||||
case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000:
|
||||
return errors.New("outbox cleanup batch size must be between 1 and 1000")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Run(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
pollTicker := time.NewTicker(d.config.PollInterval)
|
||||
recoveryTicker := time.NewTicker(d.config.RecoveryInterval)
|
||||
cleanupTicker := time.NewTicker(d.config.CleanupInterval)
|
||||
defer pollTicker.Stop()
|
||||
defer recoveryTicker.Stop()
|
||||
defer cleanupTicker.Stop()
|
||||
|
||||
if err := d.RecoverExpired(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if _, err := d.DispatchOnce(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-pollTicker.C:
|
||||
case <-recoveryTicker.C:
|
||||
if err := d.RecoverExpired(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
case <-cleanupTicker.C:
|
||||
if _, err := d.CleanupPublished(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) {
|
||||
deleted, err := d.store.DeletePublishedOutboxEvents(
|
||||
ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete published outbox events: %w", err)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("claim due outbox events: %w", err)
|
||||
}
|
||||
|
||||
published := 0
|
||||
for _, event := range events {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return published, nil
|
||||
}
|
||||
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return published, nil
|
||||
}
|
||||
d.metrics.ObservePublishFailure()
|
||||
nextAttemptAt := d.now().Add(d.config.RetryDelay)
|
||||
released, releaseErr := d.store.ReleaseOutboxEvent(
|
||||
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
|
||||
)
|
||||
if releaseErr != nil {
|
||||
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
|
||||
}
|
||||
if !released {
|
||||
return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost)
|
||||
}
|
||||
continue
|
||||
}
|
||||
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
|
||||
if err != nil {
|
||||
return published, fmt.Errorf("mark outbox event published: %w", err)
|
||||
}
|
||||
if !marked {
|
||||
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
|
||||
}
|
||||
d.metrics.ObservePublished()
|
||||
published++
|
||||
}
|
||||
return published, nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
|
||||
recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now())
|
||||
if err != nil {
|
||||
return fmt.Errorf("recover expired outbox leases: %w", err)
|
||||
}
|
||||
d.metrics.ObserveRecovered(recovered)
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObservePublished() {}
|
||||
func (nopMetrics) ObservePublishFailure() {}
|
||||
func (nopMetrics) ObserveRecovered(int) {}
|
||||
@@ -0,0 +1,207 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDispatcherPublishesAndMarksEvent(t *testing.T) {
|
||||
store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}}
|
||||
publisher := &publisherStub{messageID: "1-0"}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, published)
|
||||
require.Equal(t, []string{"event-1:lease-1"}, store.marked)
|
||||
}
|
||||
|
||||
func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) {
|
||||
event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}
|
||||
store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")}
|
||||
publisher := &publisherStub{messageID: "1-0"}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
|
||||
_, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, publisher.calls)
|
||||
|
||||
store.markErr = nil
|
||||
store.events[0].LeaseToken = "lease-2"
|
||||
publisher.messageID = "2-0"
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, published)
|
||||
require.Equal(t, 2, publisher.calls)
|
||||
require.JSONEq(t, string(event.Payload), string(publisher.payloads[0]))
|
||||
require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1]))
|
||||
require.Equal(t, []string{"event-1:lease-2"}, store.marked)
|
||||
}
|
||||
|
||||
func TestDispatcherReleasesPublishFailure(t *testing.T) {
|
||||
store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}}
|
||||
publisher := &publisherStub{err: errors.New("redis unavailable")}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, published)
|
||||
require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released)
|
||||
require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt)
|
||||
}
|
||||
|
||||
func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) {
|
||||
store := &storeStub{
|
||||
events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}},
|
||||
markLost: true,
|
||||
}
|
||||
dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"})
|
||||
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.ErrorIs(t, err, errLeaseLost)
|
||||
require.Zero(t, published)
|
||||
}
|
||||
|
||||
func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) {
|
||||
store := &storeStub{deleted: 2}
|
||||
dispatcher := newTestDispatcher(t, store, &publisherStub{})
|
||||
dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
deleted, err := dispatcher.CleanupPublished(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, deleted)
|
||||
require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore)
|
||||
require.Equal(t, 100, store.cleanupLimit)
|
||||
}
|
||||
|
||||
func TestDispatcherStopsAfterCancellation(t *testing.T) {
|
||||
store := &storeStub{}
|
||||
publisher := &publisherStub{}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- dispatcher.Run(ctx) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("dispatcher did not stop after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher {
|
||||
t.Helper()
|
||||
dispatcher, err := New(store, publisher, nil, Config{
|
||||
BatchSize: 10,
|
||||
LeaseDuration: time.Minute,
|
||||
PollInterval: time.Millisecond,
|
||||
RecoveryInterval: time.Second,
|
||||
RetryDelay: time.Second,
|
||||
Retention: 24 * time.Hour,
|
||||
CleanupInterval: time.Hour,
|
||||
CleanupBatchSize: 100,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return dispatcher
|
||||
}
|
||||
|
||||
func testPayload() json.RawMessage {
|
||||
return json.RawMessage(`{
|
||||
"schema_version": 1,
|
||||
"job_id": "11111111-1111-4111-8111-111111111111",
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": "22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-10-22T12:00:00Z"
|
||||
}`)
|
||||
}
|
||||
|
||||
type storeStub struct {
|
||||
mu sync.Mutex
|
||||
events []model.OutboxEvent
|
||||
markErr error
|
||||
markLost bool
|
||||
marked []string
|
||||
released string
|
||||
nextAttemptAt time.Time
|
||||
recoveredCalls int
|
||||
deleted int
|
||||
publishedBefore time.Time
|
||||
cleanupLimit int
|
||||
}
|
||||
|
||||
func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]model.OutboxEvent(nil), s.events...), nil
|
||||
}
|
||||
|
||||
func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.markErr != nil {
|
||||
return false, s.markErr
|
||||
}
|
||||
if s.markLost {
|
||||
return false, nil
|
||||
}
|
||||
s.marked = append(s.marked, id+":"+leaseToken)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *storeStub) ReleaseOutboxEvent(
|
||||
_ context.Context,
|
||||
id string,
|
||||
leaseToken string,
|
||||
code string,
|
||||
nextAttemptAt time.Time,
|
||||
) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.released = id + ":" + leaseToken + ":" + code
|
||||
s.nextAttemptAt = nextAttemptAt
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recoveredCalls++
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.publishedBefore = before
|
||||
s.cleanupLimit = limit
|
||||
return s.deleted, nil
|
||||
}
|
||||
|
||||
type publisherStub struct {
|
||||
mu sync.Mutex
|
||||
messageID string
|
||||
err error
|
||||
calls int
|
||||
payloads [][]byte
|
||||
}
|
||||
|
||||
func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.calls++
|
||||
p.payloads = append(p.payloads, append([]byte(nil), payload...))
|
||||
return p.messageID, p.err
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
const attachmentJobTopic = "attachment.jobs"
|
||||
|
||||
const attachmentJobColumns = `
|
||||
id, attachment_id, pipeline_version, attempt, status, available_at,
|
||||
lease_token, lease_owner, lease_until, error_code, created_at, updated_at`
|
||||
|
||||
func (s *Store) QueueAttachmentWithJobAndOutbox(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
stagingKey string,
|
||||
job model.AttachmentJob,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) (model.Attachment, error) {
|
||||
if job.ID == "" || job.PipelineVersion < 1 {
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
if job.Attempt == 0 {
|
||||
job.Attempt = 1
|
||||
}
|
||||
if job.Attempt != 1 {
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if job.CreatedAt.IsZero() {
|
||||
job.CreatedAt = now
|
||||
}
|
||||
if job.AvailableAt.IsZero() {
|
||||
job.AvailableAt = now
|
||||
}
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
query := `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', pipeline_version = $3, staging_key = $2,
|
||||
ready_at = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND object_key = $2 AND status = 'pending'
|
||||
RETURNING ` + attachmentColumns
|
||||
var attachment model.Attachment
|
||||
err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job)
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("queue attachment: %w", err)
|
||||
}
|
||||
|
||||
job.AttachmentID = attachmentID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = job.CreatedAt
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) QueueAttachmentPipelineVersion(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
job model.AttachmentJob,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) (model.Attachment, error) {
|
||||
if job.ID == "" || job.PipelineVersion < 2 {
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
job.Attempt = 1
|
||||
now := time.Now().UTC()
|
||||
job.CreatedAt = now
|
||||
job.AvailableAt = now
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE`
|
||||
var attachment model.Attachment
|
||||
if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err)
|
||||
}
|
||||
if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 {
|
||||
return model.Attachment{}, ErrConflict
|
||||
}
|
||||
job.AttachmentID = attachment.ID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = now
|
||||
const queueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', pipeline_version = $2, staging_key = object_key,
|
||||
ready_at = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1
|
||||
RETURNING ` + attachmentColumns
|
||||
if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err)
|
||||
}
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) alreadyQueuedAttachment(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
attachmentID string,
|
||||
stagingKey string,
|
||||
job model.AttachmentJob,
|
||||
) (model.Attachment, error) {
|
||||
query := `
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachments a
|
||||
JOIN attachment_jobs j
|
||||
ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
|
||||
WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready')
|
||||
AND j.id = $3 AND j.pipeline_version = $4`
|
||||
var attachment model.Attachment
|
||||
if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrConflict
|
||||
}
|
||||
return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error {
|
||||
const query = `
|
||||
INSERT INTO attachment_jobs (
|
||||
id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt,
|
||||
); err != nil {
|
||||
return fmt.Errorf("insert attachment job: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertAttachmentJobOutbox(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
job model.AttachmentJob,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) error {
|
||||
payload, err := json.Marshal(model.AttachmentJobMessage{
|
||||
SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1",
|
||||
AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt, CreatedAt: job.CreatedAt,
|
||||
Traceparent: traceparent, Tracestate: tracestate,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal attachment job event: %w", err)
|
||||
}
|
||||
const query = `
|
||||
INSERT INTO outbox_events (
|
||||
id, topic, partition_key, payload, job_id, attempt, available_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload,
|
||||
job.ID, job.Attempt, job.AvailableAt,
|
||||
); err != nil {
|
||||
return fmt.Errorf("insert attachment job outbox event: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) {
|
||||
const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1`
|
||||
var job model.AttachmentJob
|
||||
if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.AttachmentJob{}, ErrNotFound
|
||||
}
|
||||
return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Store) ClaimAttachmentJob(
|
||||
ctx context.Context,
|
||||
jobID string,
|
||||
attachmentID string,
|
||||
pipelineVersion int,
|
||||
attempt int,
|
||||
workerID string,
|
||||
leaseDuration time.Duration,
|
||||
) (model.AttachmentJobLease, error) {
|
||||
if leaseDuration <= 0 || workerID == "" {
|
||||
return model.AttachmentJobLease{}, model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
||||
if err != nil {
|
||||
return model.AttachmentJobLease{}, err
|
||||
}
|
||||
if attachment.ID != attachmentID || job.AttachmentID != attachmentID {
|
||||
return model.AttachmentJobLease{}, model.ErrInvalidInput
|
||||
}
|
||||
if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion {
|
||||
return model.AttachmentJobLease{}, model.ErrStalePipeline
|
||||
}
|
||||
if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady {
|
||||
return model.AttachmentJobLease{}, model.ErrJobComplete
|
||||
}
|
||||
if job.Attempt != attempt {
|
||||
return model.AttachmentJobLease{}, model.ErrStaleAttempt
|
||||
}
|
||||
if job.Status == model.AttachmentJobProcessing {
|
||||
return model.AttachmentJobLease{}, model.ErrJobBusy
|
||||
}
|
||||
if job.Status != model.AttachmentJobQueued {
|
||||
return model.AttachmentJobLease{}, ErrConflict
|
||||
}
|
||||
if job.AvailableAt.After(time.Now()) {
|
||||
return model.AttachmentJobLease{}, model.ErrJobNotDue
|
||||
}
|
||||
|
||||
job.LeaseToken = uuid.NewString()
|
||||
job.LeaseOwner = workerID
|
||||
leaseUntil := time.Now().UTC().Add(leaseDuration)
|
||||
job.LeaseUntil = &leaseUntil
|
||||
job.Status = model.AttachmentJobProcessing
|
||||
const updateJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing', lease_token = $2, lease_owner = $3,
|
||||
lease_until = $4, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err)
|
||||
}
|
||||
const updateAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'`
|
||||
result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion)
|
||||
if err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err)
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return model.AttachmentJobLease{}, model.ErrStalePipeline
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err)
|
||||
}
|
||||
attachment.Status = model.AttachmentProcessing
|
||||
attachment.ErrorCode = nil
|
||||
return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil
|
||||
}
|
||||
|
||||
func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) {
|
||||
query := `
|
||||
SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = $1
|
||||
FOR UPDATE OF j, a`
|
||||
var job model.AttachmentJob
|
||||
var attachment model.Attachment
|
||||
var leaseToken, leaseOwner *string
|
||||
var stagingKey *string
|
||||
err := tx.QueryRow(ctx, query, jobID).Scan(
|
||||
&job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt,
|
||||
&leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt,
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.AttachmentJob{}, model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err)
|
||||
}
|
||||
if leaseToken != nil {
|
||||
job.LeaseToken = *leaseToken
|
||||
}
|
||||
if leaseOwner != nil {
|
||||
job.LeaseOwner = *leaseOwner
|
||||
}
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return job, attachment, nil
|
||||
}
|
||||
|
||||
func prefixedAttachmentJobColumns(alias string) string {
|
||||
return fmt.Sprintf(
|
||||
"%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+
|
||||
"%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+
|
||||
"%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at",
|
||||
alias,
|
||||
)
|
||||
}
|
||||
|
||||
func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error {
|
||||
var leaseToken, leaseOwner *string
|
||||
err := row.Scan(
|
||||
&job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt,
|
||||
&leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt,
|
||||
)
|
||||
if leaseToken != nil {
|
||||
job.LeaseToken = *leaseToken
|
||||
}
|
||||
if leaseOwner != nil {
|
||||
job.LeaseOwner = *leaseOwner
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) CompleteAttachmentJob(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
variants []model.Variant,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validCompletedVariants(variants, lease.Job.PipelineVersion) {
|
||||
return "", model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin complete attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
|
||||
if err != nil || result != model.JobMutationApplied {
|
||||
return result, err
|
||||
}
|
||||
const deleteVariants = `
|
||||
DELETE FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return "", fmt.Errorf("replace attachment variants: %w", err)
|
||||
}
|
||||
|
||||
var originalKey string
|
||||
const insertVariant = `
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, pipeline_version, name, content_type, size_bytes,
|
||||
sha256, width, height, object_key
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`
|
||||
for _, variant := range variants {
|
||||
if variant.Name == "original" {
|
||||
originalKey = variant.ObjectKey
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name,
|
||||
variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width,
|
||||
variant.Height, variant.ObjectKey,
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
const completeJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2`
|
||||
if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil {
|
||||
return "", fmt.Errorf("complete attachment job: %w", err)
|
||||
}
|
||||
const completeAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'ready', object_key = $3, ready_at = clock_timestamp(),
|
||||
staging_key = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil {
|
||||
return "", fmt.Errorf("complete attachment: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit completed attachment job: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func validCompletedVariants(variants []model.Variant, pipelineVersion int) bool {
|
||||
if len(variants) != 4 || pipelineVersion < 1 {
|
||||
return false
|
||||
}
|
||||
required := map[string]struct{}{
|
||||
"original": {}, "thumbnail": {}, "crop": {}, "optimized": {},
|
||||
}
|
||||
for _, variant := range variants {
|
||||
if variant.ObjectKey == "" || variant.PipelineVersion != pipelineVersion {
|
||||
return false
|
||||
}
|
||||
if _, exists := required[variant.Name]; !exists {
|
||||
return false
|
||||
}
|
||||
delete(required, variant.Name)
|
||||
}
|
||||
return len(required) == 0
|
||||
}
|
||||
|
||||
func (s *Store) RetryAttachmentJob(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
errorCode string,
|
||||
nextAttemptAt time.Time,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
|
||||
return "", model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin retry attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
|
||||
if err != nil || result != model.JobMutationApplied {
|
||||
return result, err
|
||||
}
|
||||
job.Attempt++
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.AvailableAt = nextAttemptAt.UTC()
|
||||
job.ErrorCode = &errorCode
|
||||
const retryJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $3, available_at = $4,
|
||||
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = $5, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2`
|
||||
if _, err := tx.Exec(
|
||||
ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode,
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("retry attachment job: %w", err)
|
||||
}
|
||||
const queueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return "", fmt.Errorf("requeue attachment: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit attachment job retry: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func (s *Store) FailAttachmentJob(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
errorCode string,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validErrorCode(errorCode) {
|
||||
return "", model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin fail attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
|
||||
if err != nil || result != model.JobMutationApplied {
|
||||
return result, err
|
||||
}
|
||||
const failJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = $3, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2`
|
||||
if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil {
|
||||
return "", fmt.Errorf("fail attachment job: %w", err)
|
||||
}
|
||||
const failAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
||||
return "", fmt.Errorf("fail attachment: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit failed attachment job: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func lockJobMutation(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
lease model.AttachmentJobLease,
|
||||
) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) {
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID)
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, model.Attachment{}, "", err
|
||||
}
|
||||
if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion {
|
||||
return job, attachment, model.JobMutationStale, nil
|
||||
}
|
||||
if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady {
|
||||
return job, attachment, model.JobMutationAlreadyComplete, nil
|
||||
}
|
||||
if job.PipelineVersion != lease.Job.PipelineVersion ||
|
||||
job.Attempt != lease.Job.Attempt ||
|
||||
job.Status != model.AttachmentJobProcessing ||
|
||||
job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken ||
|
||||
job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) {
|
||||
return job, attachment, model.JobMutationStale, nil
|
||||
}
|
||||
return job, attachment, model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func validErrorCode(errorCode string) bool {
|
||||
return len(errorCode) >= 1 && len(errorCode) <= 64
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RenewAttachmentJobLease(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" {
|
||||
return false, model.ErrInvalidInput
|
||||
}
|
||||
const query = `
|
||||
UPDATE attachment_jobs
|
||||
SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2 AND lease_owner = $3
|
||||
AND status = 'processing' AND lease_until > clock_timestamp()`
|
||||
result, err := s.pool.Exec(
|
||||
ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(),
|
||||
)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("renew attachment job lease: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RecoverExpiredAttachmentJobs(
|
||||
ctx context.Context,
|
||||
limit int,
|
||||
maxAttempts int,
|
||||
retryAt time.Time,
|
||||
) (model.RecoveryResult, error) {
|
||||
if limit < 1 || limit > 1000 || maxAttempts < 1 {
|
||||
return model.RecoveryResult{}, model.ErrInvalidInput
|
||||
}
|
||||
if retryAt.IsZero() {
|
||||
retryAt = time.Now().UTC()
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
const selectExpired = `
|
||||
SELECT id
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'processing' AND lease_until <= clock_timestamp()
|
||||
ORDER BY lease_until, id
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED`
|
||||
rows, err := tx.Query(ctx, selectExpired, limit)
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err)
|
||||
}
|
||||
jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err)
|
||||
}
|
||||
|
||||
var recovered model.RecoveryResult
|
||||
for _, jobID := range jobIDs {
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) {
|
||||
continue
|
||||
}
|
||||
if job.PipelineVersion != attachment.PipelineVersion {
|
||||
if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Failed++
|
||||
continue
|
||||
}
|
||||
if job.Attempt >= maxAttempts {
|
||||
if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Failed++
|
||||
continue
|
||||
}
|
||||
|
||||
job.Attempt++
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.AvailableAt = retryAt.UTC()
|
||||
const requeueJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2, available_at = $3,
|
||||
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = 'lease_expired', updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err)
|
||||
}
|
||||
const requeueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Requeued++
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err)
|
||||
}
|
||||
return recovered, nil
|
||||
}
|
||||
|
||||
func markExpiredJobFailed(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
job model.AttachmentJob,
|
||||
attachment model.Attachment,
|
||||
errorCode string,
|
||||
failAttachment bool,
|
||||
) error {
|
||||
const failJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = $2, updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
|
||||
return fmt.Errorf("fail expired attachment job: %w", err)
|
||||
}
|
||||
if !failAttachment {
|
||||
return nil
|
||||
}
|
||||
const failAttachmentQuery = `
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
||||
return fmt.Errorf("fail attachment after expired lease: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ReplayFailedAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
actor string,
|
||||
reason string,
|
||||
) (model.AttachmentJob, error) {
|
||||
if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 {
|
||||
return model.AttachmentJob{}, model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
const selectJob = `
|
||||
SELECT j.id
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed'
|
||||
FOR UPDATE OF j, a`
|
||||
var jobID string
|
||||
if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.AttachmentJob{}, ErrConflict
|
||||
}
|
||||
return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err)
|
||||
}
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, err
|
||||
}
|
||||
previousAttempt := job.Attempt
|
||||
job.Attempt++
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.AvailableAt = time.Now().UTC()
|
||||
job.ErrorCode = nil
|
||||
job.LeaseToken = ""
|
||||
job.LeaseOwner = ""
|
||||
job.LeaseUntil = nil
|
||||
const replayJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2, available_at = $3,
|
||||
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err)
|
||||
}
|
||||
const replayAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err)
|
||||
}
|
||||
const insertAudit = `
|
||||
INSERT INTO attachment_replay_audit (
|
||||
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID,
|
||||
previousAttempt, job.Attempt, actor, reason,
|
||||
); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
return model.AttachmentJob{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
const attachmentColumns = `
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, created_at, updated_at, ready_at`
|
||||
object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key`
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(...any) error
|
||||
@@ -74,7 +74,7 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachments a
|
||||
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
|
||||
WHERE a.id = $1 AND a.status = 'ready'`
|
||||
WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')`
|
||||
|
||||
var attachment model.Attachment
|
||||
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
|
||||
@@ -84,9 +84,68 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
|
||||
}
|
||||
if attachment.Status == model.AttachmentReady {
|
||||
attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion)
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentVariantForUser(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
userID string,
|
||||
variantName string,
|
||||
) (model.Attachment, model.Variant, error) {
|
||||
attachment, err := s.AttachmentForUser(ctx, attachmentID, userID)
|
||||
if err != nil {
|
||||
return model.Attachment{}, model.Variant{}, err
|
||||
}
|
||||
if attachment.Status != model.AttachmentReady {
|
||||
return model.Attachment{}, model.Variant{}, ErrConflict
|
||||
}
|
||||
for _, variant := range attachment.Variants {
|
||||
if variant.Name == variantName {
|
||||
return attachment, variant, nil
|
||||
}
|
||||
}
|
||||
if variantName == "original" && attachment.ObjectKey != "" {
|
||||
return attachment, model.Variant{
|
||||
Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes,
|
||||
SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey,
|
||||
PipelineVersion: attachment.PipelineVersion,
|
||||
}, nil
|
||||
}
|
||||
return model.Attachment{}, model.Variant{}, ErrNotFound
|
||||
}
|
||||
|
||||
func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) {
|
||||
const query = `
|
||||
SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY CASE name
|
||||
WHEN 'original' THEN 0
|
||||
WHEN 'thumbnail' THEN 1
|
||||
WHEN 'crop' THEN 2
|
||||
WHEN 'optimized' THEN 3
|
||||
ELSE 4
|
||||
END`
|
||||
rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list attachment variants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan attachment variants: %w", err)
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
|
||||
|
||||
@@ -102,7 +161,12 @@ func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
|
||||
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
|
||||
const query = `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_variants WHERE object_key = $1
|
||||
)`
|
||||
var exists bool
|
||||
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
|
||||
return false, fmt.Errorf("check attachment by object key: %w", err)
|
||||
@@ -110,16 +174,51 @@ func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey strin
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentObjectKeys(ctx context.Context, attachmentID string) ([]string, error) {
|
||||
const query = `
|
||||
SELECT object_key
|
||||
FROM (
|
||||
SELECT object_key
|
||||
FROM attachments
|
||||
WHERE id = $1 AND object_key <> ''
|
||||
UNION
|
||||
SELECT object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1
|
||||
) objects
|
||||
ORDER BY object_key`
|
||||
rows, err := s.pool.Query(ctx, query, attachmentID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list attachment object keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
keys, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan attachment object keys: %w", err)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) {
|
||||
if limit < 1 || limit > 1000 {
|
||||
return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000")
|
||||
}
|
||||
query := `
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `,
|
||||
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
|
||||
FROM attachments a
|
||||
WHERE a.object_key > $1
|
||||
ORDER BY a.object_key
|
||||
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id),
|
||||
objects.object_key
|
||||
FROM (
|
||||
SELECT id AS attachment_id, object_key
|
||||
FROM attachments
|
||||
WHERE object_key <> ''
|
||||
UNION
|
||||
SELECT attachment_id, object_key
|
||||
FROM attachment_variants
|
||||
) objects
|
||||
JOIN attachments a ON a.id = objects.attachment_id
|
||||
WHERE objects.object_key > $1
|
||||
ORDER BY objects.object_key
|
||||
LIMIT $2`
|
||||
rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1)
|
||||
if err != nil {
|
||||
@@ -129,7 +228,9 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi
|
||||
|
||||
attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) {
|
||||
var attachment model.Attachment
|
||||
err := scanAttachmentWithReference(row, &attachment)
|
||||
var objectKey string
|
||||
err := scanAttachmentWithReferenceAndObjectKey(row, &attachment, &objectKey)
|
||||
attachment.ObjectKey = objectKey
|
||||
return publicAttachment(attachment), err
|
||||
}))
|
||||
if err != nil {
|
||||
@@ -143,13 +244,38 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi
|
||||
return attachments, next, nil
|
||||
}
|
||||
|
||||
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
|
||||
return row.Scan(
|
||||
func scanAttachmentWithReferenceAndObjectKey(
|
||||
row rowScanner,
|
||||
attachment *model.Attachment,
|
||||
objectKey *string,
|
||||
) error {
|
||||
var stagingKey *string
|
||||
err := row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
&attachment.Referenced, objectKey,
|
||||
)
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
|
||||
var stagingKey *string
|
||||
err := row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
&attachment.Referenced,
|
||||
)
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
|
||||
@@ -232,23 +358,36 @@ func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID str
|
||||
}
|
||||
|
||||
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
|
||||
return row.Scan(
|
||||
var stagingKey *string
|
||||
err := row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
)
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func attachmentColumnsWithAlias(alias string) string {
|
||||
return fmt.Sprintf(
|
||||
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
|
||||
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
|
||||
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
|
||||
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+
|
||||
"%[1]s.error_code, %[1]s.staging_key",
|
||||
alias,
|
||||
)
|
||||
}
|
||||
|
||||
func publicAttachment(attachment model.Attachment) model.Attachment {
|
||||
attachment.Variants = []model.Variant{}
|
||||
variants := make([]model.Variant, 0, len(attachment.Variants))
|
||||
for _, variant := range attachment.Variants {
|
||||
if variant.Name != "original" {
|
||||
variants = append(variants, variant)
|
||||
}
|
||||
}
|
||||
attachment.Variants = variants
|
||||
return attachment
|
||||
}
|
||||
|
||||
@@ -8,4 +8,10 @@ var (
|
||||
ErrInvalidMembers = model.ErrInvalidMembers
|
||||
ErrInvalidCursor = model.ErrInvalidCursor
|
||||
ErrIdempotencyConflict = model.ErrIdempotencyConflict
|
||||
ErrStaleLease = model.ErrStaleLease
|
||||
ErrStaleAttempt = model.ErrStaleAttempt
|
||||
ErrStalePipeline = model.ErrStalePipeline
|
||||
ErrJobBusy = model.ErrJobBusy
|
||||
ErrJobComplete = model.ErrJobComplete
|
||||
ErrJobNotDue = model.ErrJobNotDue
|
||||
)
|
||||
|
||||
@@ -51,13 +51,16 @@ func (s *Store) CreateMessage(
|
||||
if !visible {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||
validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if !validAttachments {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
if failedAttachment {
|
||||
return model.Message{}, false, ErrConflict
|
||||
}
|
||||
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
@@ -92,28 +95,35 @@ func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte,
|
||||
return sha256.Sum256(payload), nil
|
||||
}
|
||||
|
||||
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
|
||||
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) {
|
||||
if len(attachmentIDs) == 0 {
|
||||
return true, nil
|
||||
return true, false, nil
|
||||
}
|
||||
const query = `
|
||||
SELECT id
|
||||
SELECT id, status
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
|
||||
WHERE chat_id = $1 AND id::text = ANY($2)
|
||||
AND status IN ('queued', 'processing', 'ready', 'failed')
|
||||
FOR UPDATE`
|
||||
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
return false, false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
count := 0
|
||||
var count int
|
||||
var failed bool
|
||||
for rows.Next() {
|
||||
var id, status string
|
||||
if err := rows.Scan(&id, &status); err != nil {
|
||||
return false, false, fmt.Errorf("scan message attachment: %w", err)
|
||||
}
|
||||
count++
|
||||
failed = failed || status == model.AttachmentFailed
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
return false, false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
return count == len(attachmentIDs), nil
|
||||
return count == len(attachmentIDs), failed, nil
|
||||
}
|
||||
|
||||
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) ClaimOutboxEvents(
|
||||
ctx context.Context,
|
||||
limit int,
|
||||
leaseDuration time.Duration,
|
||||
) ([]model.OutboxEvent, error) {
|
||||
if limit < 1 || limit > 1000 || leaseDuration <= 0 {
|
||||
return nil, model.ErrInvalidInput
|
||||
}
|
||||
claimToken := uuid.NewString()
|
||||
query := `
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM outbox_events
|
||||
WHERE published_at IS NULL
|
||||
AND available_at <= clock_timestamp()
|
||||
AND (claim_until IS NULL OR claim_until <= clock_timestamp())
|
||||
ORDER BY available_at, id
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE outbox_events event
|
||||
SET claim_token = $2,
|
||||
claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond',
|
||||
publish_tries = event.publish_tries + 1
|
||||
FROM candidates
|
||||
WHERE event.id = candidates.id
|
||||
RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id,
|
||||
event.attempt, event.available_at, event.publish_tries, event.claim_token,
|
||||
event.claim_until, event.last_error_code, event.created_at`
|
||||
rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("claim outbox events: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) {
|
||||
var event model.OutboxEvent
|
||||
err := row.Scan(
|
||||
&event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID,
|
||||
&event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken,
|
||||
&event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt,
|
||||
)
|
||||
return event, err
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan claimed outbox events: %w", err)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) {
|
||||
const query = `
|
||||
UPDATE outbox_events
|
||||
SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL,
|
||||
last_error_code = NULL
|
||||
WHERE id = $1 AND claim_token = $2 AND published_at IS NULL
|
||||
AND claim_until > clock_timestamp()`
|
||||
result, err := s.pool.Exec(ctx, query, eventID, leaseToken)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("mark outbox event published: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func (s *Store) ReleaseOutboxEvent(
|
||||
ctx context.Context,
|
||||
eventID string,
|
||||
leaseToken string,
|
||||
errorCode string,
|
||||
nextAttemptAt time.Time,
|
||||
) (bool, error) {
|
||||
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
|
||||
return false, model.ErrInvalidInput
|
||||
}
|
||||
const query = `
|
||||
UPDATE outbox_events
|
||||
SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4
|
||||
WHERE id = $1 AND claim_token = $2 AND published_at IS NULL
|
||||
AND claim_until > clock_timestamp()`
|
||||
result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("release outbox event: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
const query = `
|
||||
UPDATE outbox_events
|
||||
SET claim_token = NULL, claim_until = NULL
|
||||
WHERE published_at IS NULL AND claim_until <= $1`
|
||||
result, err := s.pool.Exec(ctx, query, now.UTC())
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("recover expired outbox events: %w", err)
|
||||
}
|
||||
return int(result.RowsAffected()), nil
|
||||
}
|
||||
|
||||
func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) {
|
||||
if publishedBefore.IsZero() || limit < 1 || limit > 1000 {
|
||||
return 0, model.ErrInvalidInput
|
||||
}
|
||||
const query = `
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM outbox_events
|
||||
WHERE published_at < $1
|
||||
ORDER BY published_at, id
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
DELETE FROM outbox_events event
|
||||
USING candidates
|
||||
WHERE event.id = candidates.id`
|
||||
result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete published outbox events: %w", err)
|
||||
}
|
||||
return int(result.RowsAffected()), nil
|
||||
}
|
||||
@@ -2,7 +2,9 @@ package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -159,6 +161,224 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
|
||||
s.False(deleted)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
s.Equal(model.AttachmentQueued, attachment.Status)
|
||||
|
||||
events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotEmpty(events)
|
||||
var event model.OutboxEvent
|
||||
for _, candidate := range events {
|
||||
if candidate.JobID == job.ID {
|
||||
event = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
s.Require().NotEmpty(event.ID)
|
||||
var message model.AttachmentJobMessage
|
||||
s.Require().NoError(json.Unmarshal(event.Payload, &message))
|
||||
s.Equal(job.ID, message.JobID)
|
||||
s.Equal(attachment.ID, message.AttachmentID)
|
||||
s.Equal(1, message.Attempt)
|
||||
published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken)
|
||||
s.Require().NoError(err)
|
||||
s.True(published)
|
||||
|
||||
_, replay, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "queued attachment",
|
||||
[]string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.False(replay)
|
||||
|
||||
_, err = s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute,
|
||||
)
|
||||
s.ErrorIs(err, model.ErrInvalidInput)
|
||||
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentProcessing, lease.Attachment.Status)
|
||||
s.NotEmpty(lease.Attachment.StagingKey)
|
||||
|
||||
variants := completedVariants(attachment.ID, 1)
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationAlreadyComplete, result)
|
||||
|
||||
metadata, variant, err := s.store.AttachmentVariantForUser(
|
||||
context.Background(), attachment.ID, owner.ID, "thumbnail",
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentReady, metadata.Status)
|
||||
s.Len(metadata.Variants, 3)
|
||||
s.Equal(variants[1].ObjectKey, variant.ObjectKey)
|
||||
listedKeys := make(map[string]struct{})
|
||||
var after string
|
||||
for {
|
||||
listed, next, err := s.store.ListAttachments(context.Background(), after, 1000)
|
||||
s.Require().NoError(err)
|
||||
for _, candidate := range listed {
|
||||
if candidate.ID == attachment.ID {
|
||||
listedKeys[candidate.ObjectKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
if next == "" {
|
||||
break
|
||||
}
|
||||
after = next
|
||||
}
|
||||
for _, candidate := range variants {
|
||||
s.Contains(listedKeys, candidate.ObjectKey)
|
||||
}
|
||||
|
||||
v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2}
|
||||
queuedV2, err := s.store.QueueAttachmentPipelineVersion(
|
||||
context.Background(), attachment.ID, v2, "", "",
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentQueued, queuedV2.Status)
|
||||
s.Equal(2, queuedV2.PipelineVersion)
|
||||
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
_, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
_, secondJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
events, err := s.store.ClaimOutboxEvents(context.Background(), 1000, time.Minute)
|
||||
s.Require().NoError(err)
|
||||
|
||||
eventIDs := make([]string, 0, 2)
|
||||
for _, event := range events {
|
||||
if event.JobID != firstJob.ID && event.JobID != secondJob.ID {
|
||||
continue
|
||||
}
|
||||
published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken)
|
||||
s.Require().NoError(err)
|
||||
s.True(published)
|
||||
eventIDs = append(eventIDs, event.ID)
|
||||
}
|
||||
s.Require().Len(eventIDs, 2)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE outbox_events SET published_at = clock_timestamp() - INTERVAL '2 days' WHERE id IN ($1, $2)`,
|
||||
eventIDs[0], eventIDs[1],
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
|
||||
deleted, err := s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, deleted)
|
||||
deleted, err = s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, deleted)
|
||||
|
||||
var remaining int
|
||||
err = s.pool.QueryRow(
|
||||
context.Background(), `SELECT count(*) FROM outbox_events WHERE id IN ($1, $2)`, eventIDs[0], eventIDs[1],
|
||||
).Scan(&remaining)
|
||||
s.Require().NoError(err)
|
||||
s.Zero(remaining)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image")
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
|
||||
_, _, err = s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "failed attachment",
|
||||
[]string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.ErrorIs(err, postgres.ErrConflict)
|
||||
metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentFailed, metadata.Status)
|
||||
s.Require().NotNil(metadata.ErrorCode)
|
||||
s.Equal("invalid_image", *metadata.ErrorCode)
|
||||
|
||||
replayed, err := s.store.ReplayFailedAttachment(
|
||||
context.Background(), attachment.ID, "operator@example", "input repaired",
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, replayed.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, replayed.Status)
|
||||
|
||||
var auditCount int
|
||||
err = s.pool.QueryRow(
|
||||
context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID,
|
||||
).Scan(&auditCount)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, auditCount)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
|
||||
job.ID,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now())
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, recovered.Requeued)
|
||||
s.Zero(recovered.Failed)
|
||||
current, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, current.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, current.Status)
|
||||
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1))
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
|
||||
complete := completedVariants(attachment.ID, job.PipelineVersion)
|
||||
_, err = s.store.CompleteAttachmentJob(context.Background(), lease, complete[:3])
|
||||
s.ErrorIs(err, model.ErrInvalidInput)
|
||||
|
||||
duplicate := append([]model.Variant(nil), complete...)
|
||||
duplicate[3].Name = duplicate[2].Name
|
||||
_, err = s.store.CompleteAttachmentJob(context.Background(), lease, duplicate)
|
||||
s.ErrorIs(err, model.ErrInvalidInput)
|
||||
|
||||
mismatched := append([]model.Variant(nil), complete...)
|
||||
mismatched[3].PipelineVersion++
|
||||
_, err = s.store.CompleteAttachmentJob(context.Background(), lease, mismatched)
|
||||
s.ErrorIs(err, model.ErrInvalidInput)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
@@ -344,6 +564,58 @@ func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string
|
||||
return owner, chat.ID
|
||||
}
|
||||
|
||||
func createQueuedAttachment(
|
||||
t *testing.T,
|
||||
store *postgres.Store,
|
||||
ownerID string,
|
||||
chatID string,
|
||||
) (model.Attachment, model.AttachmentJob) {
|
||||
t.Helper()
|
||||
attachmentID := uuid.NewString()
|
||||
stagingKey := "staging/" + attachmentID + "/original"
|
||||
pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
|
||||
ObjectKey: stagingKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreatePendingAttachment() error = %v", err)
|
||||
}
|
||||
job := model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1,
|
||||
}
|
||||
queued, err := store.QueueAttachmentWithJobAndOutbox(
|
||||
context.Background(), pending.ID, stagingKey, job, "", "",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err)
|
||||
}
|
||||
return queued, job
|
||||
}
|
||||
|
||||
func completedVariants(attachmentID string, pipelineVersion int) []model.Variant {
|
||||
types := []struct {
|
||||
name string
|
||||
contentType string
|
||||
extension string
|
||||
}{
|
||||
{name: "original", contentType: "image/png", extension: "png"},
|
||||
{name: "thumbnail", contentType: "image/jpeg", extension: "jpg"},
|
||||
{name: "crop", contentType: "image/jpeg", extension: "jpg"},
|
||||
{name: "optimized", contentType: "image/webp", extension: "webp"},
|
||||
}
|
||||
variants := make([]model.Variant, 0, len(types))
|
||||
for index, variant := range types {
|
||||
variants = append(variants, model.Variant{
|
||||
Name: variant.name, ContentType: variant.contentType, SizeBytes: int64(128 - index*16),
|
||||
SHA256: strings.Repeat(string(rune('a'+index)), 64), Width: 320 - index*32, Height: 200 - index*20,
|
||||
ObjectKey: fmt.Sprintf("attachments/%s/v%d/%s.%s", attachmentID, pipelineVersion, variant.name, variant.extension),
|
||||
PipelineVersion: pipelineVersion,
|
||||
})
|
||||
}
|
||||
return variants
|
||||
}
|
||||
|
||||
func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int {
|
||||
t.Helper()
|
||||
var count int
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
type MetadataStore interface {
|
||||
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
|
||||
AttachmentObjectKeys(context.Context, string) ([]string, error)
|
||||
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
|
||||
ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error)
|
||||
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
|
||||
@@ -23,6 +24,7 @@ type ObjectStore interface {
|
||||
|
||||
type Config struct {
|
||||
Prefix string
|
||||
OrphanPrefixes []string
|
||||
PageSize int
|
||||
GraceTime time.Duration
|
||||
UnreferencedRetention time.Duration
|
||||
@@ -65,6 +67,11 @@ func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconcile
|
||||
if config.UnreferencedRetention <= 0 {
|
||||
return nil, errors.New("unreferenced retention must be positive")
|
||||
}
|
||||
for _, prefix := range config.OrphanPrefixes {
|
||||
if prefix == "" || prefix == config.Prefix {
|
||||
return nil, errors.New("orphan prefix must be non-empty and distinct")
|
||||
}
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
@@ -113,9 +120,35 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, prefix := range r.config.OrphanPrefixes {
|
||||
if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) scanOrphanPrefix(
|
||||
ctx context.Context,
|
||||
prefix string,
|
||||
cutoff time.Time,
|
||||
report *Report,
|
||||
) error {
|
||||
objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for objects.current != nil {
|
||||
if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.advance(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handlePair(
|
||||
ctx context.Context,
|
||||
metadata model.Attachment,
|
||||
@@ -126,6 +159,9 @@ func (r *Reconciler) handlePair(
|
||||
report.MetadataScanned++
|
||||
report.ObjectsScanned++
|
||||
switch metadata.Status {
|
||||
case model.AttachmentQueued, model.AttachmentProcessing, model.AttachmentFailed:
|
||||
report.Consistent++
|
||||
return nil
|
||||
case model.AttachmentPending:
|
||||
if metadata.CreatedAt.After(anomalyCutoff) {
|
||||
report.FreshPendingAttachments++
|
||||
@@ -176,10 +212,16 @@ func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Att
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
if err := r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
|
||||
return fmt.Errorf("delete attachment object %q: %w", attachment.ObjectKey, err)
|
||||
objectKeys, err := r.metadata.AttachmentObjectKeys(ctx, attachment.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list attachment %q object keys: %w", attachment.ID, err)
|
||||
}
|
||||
for _, objectKey := range objectKeys {
|
||||
if err := r.objects.Delete(ctx, objectKey); err != nil {
|
||||
return fmt.Errorf("delete attachment object %q: %w", objectKey, err)
|
||||
}
|
||||
report.DeletedAttachmentObjects++
|
||||
}
|
||||
report.DeletedAttachmentObjects++
|
||||
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err)
|
||||
@@ -197,6 +239,14 @@ func (r *Reconciler) handleOrphanObject(
|
||||
report *Report,
|
||||
) error {
|
||||
report.ObjectsScanned++
|
||||
exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recheck orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
if exists {
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
if object.LastModified.After(cutoff) {
|
||||
report.FreshOrphanObjects++
|
||||
return nil
|
||||
@@ -205,13 +255,6 @@ func (r *Reconciler) handleOrphanObject(
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recheck orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
if err := r.objects.Delete(ctx, object.Key); err != nil {
|
||||
return fmt.Errorf("delete orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
@@ -226,6 +269,19 @@ func (r *Reconciler) handleMissingObject(
|
||||
report *Report,
|
||||
) error {
|
||||
report.MetadataScanned++
|
||||
if metadata.Status == model.AttachmentQueued ||
|
||||
metadata.Status == model.AttachmentProcessing ||
|
||||
metadata.Status == model.AttachmentFailed {
|
||||
return nil
|
||||
}
|
||||
if metadata.Status == model.AttachmentPending {
|
||||
if metadata.CreatedAt.After(cutoff) {
|
||||
report.FreshPendingAttachments++
|
||||
return nil
|
||||
}
|
||||
report.OldPendingAttachments++
|
||||
return r.deletePair(ctx, metadata, cutoff, report)
|
||||
}
|
||||
if metadata.Status == model.AttachmentDeleting {
|
||||
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
|
||||
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
|
||||
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
|
||||
"staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false),
|
||||
}}
|
||||
objects := &objectStub{items: map[string]model.StoredObject{
|
||||
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
|
||||
@@ -28,27 +29,33 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
|
||||
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
|
||||
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
|
||||
"staging/orphan/original": {Key: "staging/orphan/original", LastModified: old},
|
||||
"staging/pending/original": {Key: "staging/pending/original", LastModified: old},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
|
||||
OrphanPrefixes: []string{"staging/"},
|
||||
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, report.OldPendingAttachments)
|
||||
require.Equal(t, 2, report.OldPendingAttachments)
|
||||
require.Equal(t, 1, report.OldUnreferencedAttachments)
|
||||
require.Equal(t, 1, report.FreshUnreferencedAttachments)
|
||||
require.Equal(t, 1, report.DeletedOrphanObjects)
|
||||
require.Equal(t, 2, report.DeletedAttachmentObjects)
|
||||
require.Equal(t, 3, report.DeletedUnreferencedRows)
|
||||
require.Equal(t, 2, report.DeletedOrphanObjects)
|
||||
require.Equal(t, 3, report.DeletedAttachmentObjects)
|
||||
require.Equal(t, 4, report.DeletedUnreferencedRows)
|
||||
require.Contains(t, metadata.items, "attachments/b/original")
|
||||
require.Contains(t, metadata.items, "attachments/d/original")
|
||||
require.NotContains(t, metadata.items, "attachments/a/original")
|
||||
require.NotContains(t, metadata.items, "attachments/c/original")
|
||||
require.NotContains(t, metadata.items, "attachments/missing/original")
|
||||
require.NotContains(t, metadata.items, "staging/pending/original")
|
||||
require.NotContains(t, objects.items, "attachments/orphan/original")
|
||||
require.NotContains(t, objects.items, "staging/orphan/original")
|
||||
require.NotContains(t, objects.items, "staging/pending/original")
|
||||
|
||||
repeated, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
@@ -75,6 +82,98 @@ func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) {
|
||||
require.Contains(t, objects.items, "attachments/a/original")
|
||||
}
|
||||
|
||||
func TestReconcilerDoesNotReportTrackedStagingObjectAsOrphan(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
key := "staging/a/original"
|
||||
metadata := &metadataStub{items: map[string]model.Attachment{
|
||||
key: attachment("a", key, model.AttachmentQueued, now.Add(-time.Hour), false),
|
||||
}}
|
||||
objects := &objectStub{items: map[string]model.StoredObject{
|
||||
key: {Key: key, LastModified: now.Add(-time.Hour)},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", OrphanPrefixes: []string{"staging/"}, PageSize: 1,
|
||||
GraceTime: time.Minute, UnreferencedRetention: time.Hour, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, report.OldOrphanObjects)
|
||||
require.Zero(t, report.FreshOrphanObjects)
|
||||
require.Contains(t, objects.items, key)
|
||||
}
|
||||
|
||||
func TestReconcilerPreservesAttachmentDuringReprocessing(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
|
||||
for _, status := range []string{
|
||||
model.AttachmentQueued,
|
||||
model.AttachmentProcessing,
|
||||
model.AttachmentFailed,
|
||||
} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
original := attachment("a", "attachments/a/v1/original.png", status, now.Add(-48*time.Hour), false)
|
||||
original.PipelineVersion = 2
|
||||
missing := attachment("a", "attachments/a/v1/optimized.webp", status, now.Add(-48*time.Hour), false)
|
||||
missing.PipelineVersion = 2
|
||||
metadata := &metadataStub{items: map[string]model.Attachment{
|
||||
original.ObjectKey: original,
|
||||
missing.ObjectKey: missing,
|
||||
}}
|
||||
objects := &objectStub{items: map[string]model.StoredObject{
|
||||
original.ObjectKey: {Key: original.ObjectKey, LastModified: now.Add(-48 * time.Hour)},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour,
|
||||
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, report.DeletedAttachmentObjects)
|
||||
require.Zero(t, report.DeletedUnreferencedRows)
|
||||
require.Contains(t, metadata.items, original.ObjectKey)
|
||||
require.Contains(t, metadata.items, missing.ObjectKey)
|
||||
require.Contains(t, objects.items, original.ObjectKey)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerDeletesAllAttachmentVariantsOnce(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
|
||||
old := now.Add(-48 * time.Hour)
|
||||
keys := []string{
|
||||
"attachments/a/v1/crop.jpg",
|
||||
"attachments/a/v1/optimized.webp",
|
||||
"attachments/a/v1/original.png",
|
||||
"attachments/a/v1/thumbnail.jpg",
|
||||
}
|
||||
metadata := &metadataStub{items: make(map[string]model.Attachment, len(keys))}
|
||||
objects := &objectStub{items: make(map[string]model.StoredObject, len(keys))}
|
||||
for _, key := range keys {
|
||||
metadata.items[key] = attachment("a", key, model.AttachmentReady, old, false)
|
||||
objects.items[key] = model.StoredObject{Key: key, LastModified: old}
|
||||
}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour,
|
||||
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(keys), report.DeletedAttachmentObjects)
|
||||
require.Equal(t, 1, report.DeletedUnreferencedRows)
|
||||
require.Empty(t, metadata.items)
|
||||
require.Empty(t, objects.items)
|
||||
|
||||
repeated, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, repeated.DeletedAttachmentObjects)
|
||||
require.Zero(t, repeated.DeletedUnreferencedRows)
|
||||
}
|
||||
|
||||
func TestNewRejectsInvalidRetention(t *testing.T) {
|
||||
_, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1})
|
||||
require.Error(t, err)
|
||||
@@ -93,6 +192,17 @@ type metadataStub struct {
|
||||
appeared map[string]bool
|
||||
}
|
||||
|
||||
func (s *metadataStub) AttachmentObjectKeys(_ context.Context, attachmentID string) ([]string, error) {
|
||||
var keys []string
|
||||
for key, item := range s.items {
|
||||
if item.ID == attachmentID {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) {
|
||||
keys := make([]string, 0, len(s.items))
|
||||
for key := range s.items {
|
||||
@@ -122,6 +232,7 @@ func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string
|
||||
}
|
||||
|
||||
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) {
|
||||
claim := false
|
||||
for key, item := range s.items {
|
||||
if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced {
|
||||
continue
|
||||
@@ -135,19 +246,20 @@ func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate m
|
||||
}
|
||||
item.Status = model.AttachmentDeleting
|
||||
s.items[key] = item
|
||||
return true, nil
|
||||
claim = true
|
||||
}
|
||||
return false, nil
|
||||
return claim, nil
|
||||
}
|
||||
|
||||
func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) {
|
||||
deleted := false
|
||||
for key, item := range s.items {
|
||||
if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced {
|
||||
delete(s.items, key)
|
||||
return true, nil
|
||||
deleted = true
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
type objectStub struct {
|
||||
|
||||
+11
-35
@@ -1,6 +1,7 @@
|
||||
# Адрес и внутренний порт HTTP-сервера.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
MESSENGER_CONFIG_PATH=config/messenger.yaml
|
||||
|
||||
# Адреса публикации сервисов на локальной машине.
|
||||
API_BIND_ADDRESS=127.0.0.1:8080
|
||||
@@ -8,11 +9,6 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081
|
||||
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
|
||||
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
|
||||
|
||||
# Время жизни сессии, идемпотентности и используемое хранилище.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
SESSION_STORE=valkey
|
||||
|
||||
# Число API-реплик за Nginx.
|
||||
API_REPLICAS=2
|
||||
|
||||
@@ -26,6 +22,10 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge
|
||||
VALKEY_PASSWORD=local-valkey-password
|
||||
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
|
||||
|
||||
# Redis Streams используется только как устойчивый брокер заданий.
|
||||
REDIS_BROKER_PASSWORD=local-redis-broker-password
|
||||
BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0
|
||||
|
||||
# Административные данные MinIO используются только при bootstrap.
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
MINIO_ROOT_PASSWORD=local-minio-admin-password
|
||||
@@ -36,36 +36,6 @@ S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY=messenger-app
|
||||
S3_SECRET_KEY=local-messenger-s3-password
|
||||
S3_USE_TLS=false
|
||||
S3_TIMEOUT_SECONDS=5
|
||||
ATTACHMENT_MAX_BYTES=10485760
|
||||
ATTACHMENT_MAX_PIXELS=40000000
|
||||
ATTACHMENT_BODY_MEMORY_BYTES=67108864
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
|
||||
|
||||
# Размер пула и обслуживание соединений PostgreSQL.
|
||||
DB_MIN_CONNS=1
|
||||
DB_MAX_CONNS=10
|
||||
DB_MAX_CONNECTION_AGE_SECONDS=1800
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS=300
|
||||
DB_HEALTH_CHECK_SECONDS=30
|
||||
|
||||
# Таймауты зависимостей, миграций и HTTP-запросов.
|
||||
DEPENDENCY_TIMEOUT_SECONDS=5
|
||||
MIGRATION_TIMEOUT_SECONDS=30
|
||||
REQUEST_TIMEOUT_SECONDS=5
|
||||
HTTP_READ_HEADER_TIMEOUT_SECONDS=5
|
||||
HTTP_READ_TIMEOUT_SECONDS=10
|
||||
HTTP_WRITE_TIMEOUT_SECONDS=10
|
||||
HTTP_IDLE_TIMEOUT_SECONDS=60
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
|
||||
# Параметры Argon2id для новых хешей паролей.
|
||||
PASSWORD_MEMORY_KIB=65536
|
||||
PASSWORD_ITERATIONS=3
|
||||
PASSWORD_PARALLELISM=2
|
||||
PASSWORD_SALT_LENGTH=16
|
||||
PASSWORD_KEY_LENGTH=32
|
||||
|
||||
# Порт Swagger внутри конфигурации CORS API.
|
||||
SWAGGER_PORT=8081
|
||||
@@ -91,6 +61,12 @@ POSTGRES_CPUS=2
|
||||
POSTGRES_MEMORY=1g
|
||||
VALKEY_CPUS=1
|
||||
VALKEY_MEMORY=256m
|
||||
REDIS_BROKER_CPUS=1
|
||||
REDIS_BROKER_MEMORY=256m
|
||||
DISPATCHER_CPUS=0.5
|
||||
DISPATCHER_MEMORY=128m
|
||||
WORKER_CPUS=2
|
||||
WORKER_MEMORY=1g
|
||||
S3_CPUS=1
|
||||
S3_MEMORY=512m
|
||||
S3_BOOTSTRAP_CPUS=0.25
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT attachments_status,
|
||||
DROP CONSTRAINT attachments_ready_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN error_code TEXT,
|
||||
ADD COLUMN staging_key TEXT;
|
||||
|
||||
UPDATE attachments
|
||||
SET staging_key = object_key
|
||||
WHERE status = 'pending';
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status CHECK (
|
||||
status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting')
|
||||
),
|
||||
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1),
|
||||
ADD CONSTRAINT attachments_error_code CHECK (
|
||||
(status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR
|
||||
(status <> 'failed' AND error_code IS NULL)
|
||||
),
|
||||
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||
(status = 'ready' AND ready_at IS NOT NULL) OR
|
||||
(status <> 'ready')
|
||||
),
|
||||
ADD CONSTRAINT attachments_staging_key_length CHECK (
|
||||
staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_status_updated_at_idx
|
||||
ON attachments (status, updated_at, id);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
lease_token UUID,
|
||||
lease_owner TEXT,
|
||||
lease_until TIMESTAMPTZ,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1),
|
||||
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_lease CHECK (
|
||||
(status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR
|
||||
(status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL)
|
||||
),
|
||||
CONSTRAINT attachment_jobs_error_code CHECK (
|
||||
error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_due_idx
|
||||
ON attachment_jobs (available_at, id)
|
||||
WHERE status = 'queued';
|
||||
|
||||
CREATE INDEX attachment_jobs_expired_lease_idx
|
||||
ON attachment_jobs (lease_until, id)
|
||||
WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')),
|
||||
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1),
|
||||
CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1),
|
||||
CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
|
||||
);
|
||||
|
||||
CREATE TABLE outbox_events (
|
||||
id UUID PRIMARY KEY,
|
||||
topic TEXT NOT NULL,
|
||||
partition_key TEXT NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
|
||||
attempt INTEGER NOT NULL,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
publish_tries INTEGER NOT NULL DEFAULT 0,
|
||||
claim_token UUID,
|
||||
claim_until TIMESTAMPTZ,
|
||||
last_error_code TEXT,
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt),
|
||||
CONSTRAINT outbox_events_attempt CHECK (attempt >= 1),
|
||||
CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0),
|
||||
CONSTRAINT outbox_events_claim CHECK (
|
||||
(claim_token IS NULL AND claim_until IS NULL) OR
|
||||
(claim_token IS NOT NULL AND claim_until IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT outbox_events_last_error_code CHECK (
|
||||
last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX outbox_events_due_idx
|
||||
ON outbox_events (available_at, id)
|
||||
WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE attachment_replay_audit (
|
||||
id UUID PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
previous_attempt INTEGER NOT NULL,
|
||||
new_attempt INTEGER NOT NULL,
|
||||
actor TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1),
|
||||
CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128),
|
||||
CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_replay_audit_job_created_at_idx
|
||||
ON attachment_replay_audit (job_id, created_at);
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE INDEX outbox_events_published_at_idx
|
||||
ON outbox_events (published_at, id)
|
||||
WHERE published_at IS NOT NULL;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,39 @@
|
||||
messenger_attachment_jobs:
|
||||
query: |
|
||||
WITH statuses(status) AS (
|
||||
VALUES ('queued'), ('processing'), ('failed')
|
||||
)
|
||||
SELECT
|
||||
statuses.status,
|
||||
COUNT(j.id)::bigint AS count,
|
||||
COALESCE(EXTRACT(EPOCH FROM (clock_timestamp() - MIN(j.created_at))), 0)::double precision AS oldest_age_seconds
|
||||
FROM statuses
|
||||
LEFT JOIN attachment_jobs j ON j.status = statuses.status
|
||||
GROUP BY statuses.status
|
||||
metrics:
|
||||
- status:
|
||||
usage: LABEL
|
||||
description: Attachment job status.
|
||||
- count:
|
||||
usage: GAUGE
|
||||
description: Attachment jobs in this status.
|
||||
- oldest_age_seconds:
|
||||
usage: GAUGE
|
||||
description: Age of the oldest attachment job in this status.
|
||||
|
||||
messenger_outbox:
|
||||
query: |
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE published_at IS NULL)::bigint AS pending,
|
||||
COALESCE(
|
||||
EXTRACT(EPOCH FROM (clock_timestamp() - MIN(created_at) FILTER (WHERE published_at IS NULL))),
|
||||
0
|
||||
)::double precision AS oldest_pending_age_seconds
|
||||
FROM outbox_events
|
||||
metrics:
|
||||
- pending:
|
||||
usage: GAUGE
|
||||
description: Unpublished outbox events.
|
||||
- oldest_pending_age_seconds:
|
||||
usage: GAUGE
|
||||
description: Age of the oldest unpublished outbox event.
|
||||
@@ -22,6 +22,21 @@ scrape_configs:
|
||||
static_configs:
|
||||
- targets: ["valkey-exporter:9121"]
|
||||
|
||||
- job_name: redis-broker
|
||||
static_configs:
|
||||
- targets: ["redis-broker-exporter:9121"]
|
||||
|
||||
- job_name: attachment-worker
|
||||
dns_sd_configs:
|
||||
- names: ["worker"]
|
||||
type: A
|
||||
port: 9091
|
||||
refresh_interval: 2s
|
||||
|
||||
- job_name: outbox-dispatcher
|
||||
static_configs:
|
||||
- targets: ["dispatcher:9091"]
|
||||
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
static_configs:
|
||||
|
||||
+2
-2
@@ -2,8 +2,8 @@
|
||||
|
||||
set -eu
|
||||
|
||||
docker compose stop nginx api valkey postgres s3
|
||||
docker compose start postgres valkey s3
|
||||
docker compose stop nginx api worker dispatcher redis-broker valkey postgres s3
|
||||
docker compose start postgres valkey redis-broker s3
|
||||
docker compose run --rm s3-bootstrap
|
||||
docker compose run --rm migrate
|
||||
docker compose up -d --wait --wait-timeout 90
|
||||
|
||||
@@ -150,7 +150,7 @@ fi
|
||||
cp "$upstream_file" "$upstream_backup"
|
||||
|
||||
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
|
||||
compose run --rm --no-deps s3-bootstrap
|
||||
compose run --rm --no-deps migrate
|
||||
|
||||
@@ -208,7 +208,7 @@ for container in $candidate_pool; do
|
||||
done
|
||||
smoke_check
|
||||
|
||||
compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
|
||||
compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana
|
||||
|
||||
printf '%s\n' "$candidate_pool" > "$active_containers"
|
||||
cp "$manifest" "$current_manifest"
|
||||
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
state_file=$(mktemp)
|
||||
|
||||
restore_pipeline() {
|
||||
rm -f "$state_file"
|
||||
docker compose up -d dispatcher worker >/dev/null
|
||||
}
|
||||
|
||||
trap restore_pipeline EXIT INT TERM
|
||||
|
||||
docker compose stop worker dispatcher >/dev/null
|
||||
docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file"
|
||||
docker compose restart api redis-broker >/dev/null
|
||||
docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null
|
||||
docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file"
|
||||
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
restore_pipeline() {
|
||||
docker compose up -d dispatcher worker >/dev/null
|
||||
}
|
||||
|
||||
trap restore_pipeline EXIT INT TERM
|
||||
docker compose stop worker dispatcher >/dev/null
|
||||
docker compose --profile experiments run --rm lab5-experiment
|
||||
|
||||
Reference in New Issue
Block a user