ЛР 5 — БВТ2304 — Кораблев — на 5 #54
+11
-1
@@ -11,7 +11,7 @@ SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
@@ -23,3 +23,13 @@ WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
MAX_ACTIVE_JOBS=500
|
||||
JOB_LEASE_SECONDS=30
|
||||
JOB_BACKOFF_BASE=1s
|
||||
JOB_BACKOFF_MAX=60s
|
||||
WORKER_DECODE_BUDGET_BYTES=402653184
|
||||
WORKER_MEMORY=768M
|
||||
WORKER_CPUS=2.0
|
||||
WORKER_GOMEMLIMIT=600MiB
|
||||
PENDING_UPLOAD_GRACE=15m
|
||||
FAULT_INJECTION=1
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM golang:1.23.4-alpine3.21 AS build
|
||||
WORKDIR /src
|
||||
|
||||
COPY src/go.mod src/go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY src/ ./
|
||||
ENV CGO_ENABLED=0
|
||||
RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
|
||||
|
||||
FROM alpine:3.21.2 AS runtime
|
||||
RUN addgroup -S app && adduser -S -G app app
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/server /app/server
|
||||
ARG APP_BUILD=dev
|
||||
ENV APP_BUILD=$APP_BUILD
|
||||
LABEL org.opencontainers.image.revision=$APP_BUILD
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
ENV APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1
|
||||
ENTRYPOINT ["/app/server"]
|
||||
@@ -28,3 +28,42 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: secrets up down ps unit worker-stop worker-start ops track1
|
||||
|
||||
secrets:
|
||||
$(PYTHON) scripts/init_secrets.py
|
||||
|
||||
up: secrets
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
|
||||
down:
|
||||
docker compose stop
|
||||
|
||||
ps:
|
||||
docker compose ps -a
|
||||
|
||||
unit:
|
||||
docker run --rm -v "$(CURDIR)/src:/src" -w /src golang:1.23.4-alpine3.21 go test ./...
|
||||
|
||||
worker-stop:
|
||||
docker compose stop worker
|
||||
|
||||
worker-start:
|
||||
docker compose start worker
|
||||
|
||||
ops:
|
||||
$(PYTHON) scripts/lab5ops.py worker-down --output evidence/worker-down.json
|
||||
$(PYTHON) scripts/lab5ops.py api-crash --output evidence/api-crash.json
|
||||
$(PYTHON) scripts/lab5ops.py effect-crash --output evidence/effect-crash.json
|
||||
$(PYTHON) scripts/lab5ops.py ack-crash --output evidence/ack-crash.json
|
||||
$(PYTHON) scripts/lab5ops.py retry --output evidence/retry.json
|
||||
$(PYTHON) scripts/lab5ops.py replay --output evidence/replay.json
|
||||
$(PYTHON) scripts/lab5ops.py invalid --output evidence/invalid.json
|
||||
$(PYTHON) scripts/lab5ops.py hang --output evidence/hang.json
|
||||
$(PYTHON) scripts/lab5ops.py dlq --output evidence/dlq.json
|
||||
$(PYTHON) scripts/lab5ops.py limits --output evidence/limits.json
|
||||
|
||||
track1:
|
||||
$(PYTHON) scripts/track1/run_track1.py matrix --output evidence/track1/matrix.json
|
||||
$(PYTHON) scripts/track1/run_track1.py backpressure --output evidence/track1/backpressure.json
|
||||
|
||||
@@ -1,37 +1,193 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Кораблев Петр, БВТ2304
|
||||
- Целевая оценка: **5** (уровень 3 + уровень 4 + **трек 1 «Очередь под перегрузкой»**)
|
||||
- Commit/PR: ветка `solution/BVT2304-korablev`
|
||||
- Использованные источники, библиотеки и помощь ИИ: Go 1.23.4, pgx/v5 5.7.2, go-redis/v9 9.7.0, minio-go/v7 7.0.82 (из lab4); **новые**: `github.com/rabbitmq/amqp091-go` 1.10.0 (AMQP-клиент), `golang.org/x/image` 0.23.0 (`draw.CatmullRom` для ресайза), `github.com/gen2brain/webp` 0.5.2 (libwebp, скомпилированный в WebAssembly и исполняемый через wazero 1.8.1 — кодирование WebP без cgo). Образы: PostgreSQL 17.2-alpine3.21, Valkey 8.0.2-alpine3.21, HAProxy 3.0.5-alpine, SeaweedFS 3.80, **RabbitMQ 4.0.5-management-alpine**, golang:1.23.4-alpine3.21 → alpine:3.21.2, для служебных проверок python:3.12.8-alpine3.21. Документация: RabbitMQ Reliability / Confirms / Quorum queues, EXIF 2.32 (тег Orientation 0x0112). Помощь ИИ использовалась при написании отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker 29.x + Compose v2 (Docker Desktop/WSL2), Python 3.9+, Git. Go и make локально не нужны (сборка в Docker; без make команды ниже выполняются через `python`). На Windows тесты запускать с `PYTHONUTF8=1`.
|
||||
|
||||
```sh
|
||||
python3 scripts/init_secrets.py # = make secrets: secrets/* + .env из .env.example
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait # = make up; ждёт healthy и завершения migrate/s3-init
|
||||
make check && make test # smoke через LB http://localhost:8080
|
||||
make unit # Go unit-тесты imaging (размеры, EXIF, alpha, лимиты)
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
|
||||
docker compose up -d --no-recreate --wait api # вернуть остановленную реплику
|
||||
make ops # сценарии отказов уровней 3/4 -> evidence/*.json
|
||||
make track1 # трек 1 -> evidence/track1/*.json (~15 мин)
|
||||
```
|
||||
|
||||
- `init_secrets.py` дополнительно к lab4 создаёт `secrets/rabbitmq_auth.conf` (пользователь `messenger` со случайным паролем; монтируется как Compose secret в `/etc/rabbitmq/conf.d/20-auth.conf`, гостевой пользователь не создаётся) и `secrets/broker_url` (AMQP URL для API/worker через `BROKER_URL_FILE`). Всё в `secrets/` (в `.gitignore`/`.dockerignore`).
|
||||
- Порядок старта (`depends_on`): postgres/valkey/s3/rabbitmq healthy → `migrate` (миграции под advisory lock, теперь 0001–0004) → `s3-init` → `api` ×2 и `worker` → `lb`.
|
||||
- `FAULT_INJECTION=1` в `.env` включает учебные триггеры сбоев по имени файла `fault-*` (см. ниже). Вне стенда выставить `0`.
|
||||
- Остановка worker для демонстрации: `scripts/stop-worker.sh` / `.cmd` (= `docker compose stop worker`), запуск — `scripts/start-worker.sh` / `.cmd`.
|
||||
- Безопасная остановка: `docker compose stop` (`make down`), volumes `pgdata`, `s3data`, `mqdata` сохраняются. Намеренное удаление учебных данных: `docker compose down -v` (необратимо).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
client → HAProxy (lb, 127.0.0.1:8080) → api ×2 ──┬─ PostgreSQL: attachments, attachment_jobs, job_outbox,
|
||||
│ job_events, attachment_variants (+ всё из lab4)
|
||||
├─ S3 staging/<att_id> (оригинал до обработки)
|
||||
└─ outbox relay ──publish+confirm──► RabbitMQ
|
||||
exchange attachments
|
||||
quorum queue attachment.process.v1
|
||||
DLX → attachment.process.dlq
|
||||
worker (отдельный процесс/контейнер) ◄──prefetch=N, manual ack──┘
|
||||
├─ claim/lease в PostgreSQL → читает staging → пишет в S3 attachments/<chat>/<att> (оригинал)
|
||||
│ и variants/<chat>/<att>/v1/{thumbnail.jpg,crop.jpg,optimized.webp} → COMMIT результата → удаляет staging → ack
|
||||
├─ outbox relay (тот же код, что в API; SKIP LOCKED — несколько relay безопасны)
|
||||
└─ sweeper: истёкшие lease → queued; retention staging; зависшие pending-загрузки
|
||||
```
|
||||
|
||||
**Источник истины** — PostgreSQL: строка `attachment_jobs` определяет, нужно ли что-то делать; сообщение брокера — только «разбудить worker» с `job_id`/`attempt` (`contracts/jobs.schema.json`, без байтов, ключей и URL). Worker берёт доверенные метаданные (ключи, тип, sha256) из БД.
|
||||
|
||||
**Upload, транзакционные границы** (`src/internal/api/attachments.go`, `src/internal/store/jobs.go`):
|
||||
1. аутентификация → членство (чужой — 404) → filename → Content-Type → `Content-Length` ≤ 10 MiB → дешёвая предпроверка размера очереди (`MAX_ACTIVE_JOBS`, иначе 503 + `Retry-After: 5`) → тело через `MaxBytesReader` → magic bytes + `image.DecodeConfig` (только заголовок: >40 Мпикс → 413). **Полное декодирование и ресайз API больше не делает** — только worker, даже если worker недоступен.
|
||||
2. `INSERT attachments status='pending'` (невидима в API) → `PUT staging/<att_id>` (приватный prefix того же bucket, persistent volume S3).
|
||||
3. **Одна транзакция**: `pg_advisory_xact_lock` + повторный подсчёт активных jobs (строгая граница), `pending→queued`, `INSERT attachment_jobs`, `INSERT job_outbox`, `INSERT job_events('queued')` → COMMIT → `202 queued`.
|
||||
Значит, в момент 202 устойчиво сохранены: полный проверенный оригинал в S3 staging, метаданные, job и намерение публикации (outbox). Сообщения в брокере в этот момент может ещё не быть — это нормально.
|
||||
4. Ошибка на 2/3 → 503 и компенсация (удаляем pending-строку, затем staging-объект). Оставшееся (БД недоступна) убирает sweeper через `PENDING_UPLOAD_GRACE=15m`.
|
||||
|
||||
**Transactional outbox** (`src/internal/relay`): relay берёт `job_outbox WHERE published_at IS NULL AND available_at<=now() FOR UPDATE SKIP LOCKED`, публикует persistent-сообщение и **ждёт publisher confirm**, только потом ставит `published_at` в той же транзакции. Relay работает в каждом API (сразу после COMMIT его «пинают») и в worker, плюс опрос раз в секунду. Падение между COMMIT и publish → строку опубликует любой живой relay. Падение после publish до отметки → повторная публикация, т.е. дубль сообщения, который worker обезвреживает (ниже). Недоступность RabbitMQ не ломает upload: 202 продолжает отдаваться, строки копятся в outbox.
|
||||
|
||||
**Broker** — RabbitMQ 4.0.5, quorum-очередь (реплицируемый журнал на volume `mqdata`, durable), `x-delivery-limit=20` и DLX → `attachment.process.dlq` для poison/malformed сообщений. Publisher confirms на стороне relay, manual ack на стороне worker, `prefetch = WORKER_CONCURRENCY`.
|
||||
|
||||
**Worker** (`src/internal/worker/worker.go`), на каждую доставку:
|
||||
1. Проверка схемы сообщения; некорректное → `nack(requeue=false)` → DLQ.
|
||||
2. `ClaimJob(job_id, attempt)` под `SELECT … FOR UPDATE`: `ready/failed` → no-op ack; `attempt` в сообщении ≠ текущему → устаревший дубль, ack; `processing` с живым lease → ack (работу держит другой); retry ещё не наступил → ack. Иначе `processing`, `lease_owner`, `lease_until=now()+JOB_LEASE_SECONDS`. **Redelivery** (флаг `redelivered` от RabbitMQ) разрешает перехват lease немедленно: брокер переотдаёт сообщение только после закрытия канала прежнего держателя (crash/kill), поэтому ждать истечения lease не нужно.
|
||||
3. Чтение оригинала из staging (если staging уже удалён — из конечного ключа), сверка sha256/size с БД, обработка с ограничениями (ниже), запись оригинала и вариантов **по детерминированным ключам** (`(attachment_id, pipeline_version, variant)` → один ключ S3) — повтор перезаписывает те же байты, второго набора не появляется.
|
||||
4. `CompleteJob` — одна транзакция: `UPDATE job SET ready WHERE job_id AND attempt AND status='processing'`, `INSERT attachment_variants … ON CONFLICT DO NOTHING`, `attachment → ready`. Устаревшая попытка получает 0 строк и ничего не меняет: ready никогда не откатывается.
|
||||
5. Удаление staging (после COMMIT), `staging_key=NULL` → **ack**.
|
||||
|
||||
**Ошибки и повторы**: `imaging.PermanentError` (`invalid_image`, `image_too_large`, checksum mismatch) → сразу `failed`. Временные (`storage_unavailable`, `processing_timeout`, `lease_expired`, `injected_transient`) → `queued`, `attempt+1`, новая outbox-строка с `available_at = now() + backoff`, backoff экспоненциальный с jitter «equal jitter»: `d = JOB_BACKOFF_BASE·2^(attempt-1)` (cap `JOB_BACKOFF_MAX=60s`), ожидание равномерно в `[d/2, d]`. После `JOB_MAX_ATTEMPTS=5` → `failed / retry_exhausted`. Отложенный повтор реализован через outbox, поэтому не нужен delayed-плагин RabbitMQ и повтор переживает рестарт брокера. Если сама БД недоступна при фиксации — `nack(requeue=true)`.
|
||||
|
||||
**Реестр failed и replay**: `attachment_jobs WHERE status='failed'` + аудит `job_events` (каждая попытка, причина, кто сделал replay). Явный операторский replay: `docker compose exec worker /app/server replay <attachment_id> --actor <name>` — только `failed → queued`, тот же `job_id` (тот же логический эффект), новый бюджет попыток, запись в аудит. Для `ready` replay отказывает (`noop`), для `queued/processing` — тоже; если staging уже удалён retention-ом — `refused`. DLQ брокера содержит только сообщения, которые нельзя разобрать (по ним нет job).
|
||||
|
||||
**Lease / зависший vs долгий worker**: живой worker продлевает lease (heartbeat каждые `lease/3`), но не дольше `JOB_TIMEOUT_SECONDS=60` — потом сам прерывает попытку как `processing_timeout`. Зависший (нет heartbeat) теряет lease через `JOB_LEASE_SECONDS=30`, sweeper (каждые 5 с) переводит job в `queued` с `attempt+1`. Если «оживший» старый worker всё же досчитает, его `CompleteJob` с прежним `attempt` ничего не изменит. Два одновременных исполнителя возможны только в окне сетевого разрыва; они пишут одинаковые байты по одинаковым ключам, а результат фиксирует ровно один.
|
||||
|
||||
**Варианты (pipeline_version=1)**, `src/internal/imaging/imaging.go`: EXIF Orientation (1–8) применяется до ресайза, alpha компонуется на белый, метаданные не копируются ни в один вариант (энкодерам передаются только пиксели).
|
||||
- `thumbnail`: JPEG q=85, вписан в 256×256 без увеличения, размеры округляются вниз целочисленно (320×200 → 256×160), минимум 1 px.
|
||||
- `crop`: JPEG q=85, центральный квадрат исходника → 128×128 (маленькие увеличиваются).
|
||||
- `optimized`: WebP lossy q=80, method=4, вписан в 1280×1280 без увеличения, без EXIF/ICC/XMP. Уменьшение размера маленьких файлов не обещается.
|
||||
- Ресайз — Catmull-Rom (`x/image/draw`).
|
||||
|
||||
**Лимиты worker**: ≤10 MiB bytes (повторная проверка), ≤40 Мпикс по заголовку до декодирования, общий бюджет памяти декодирования `WORKER_DECODE_BUDGET_BYTES=384 MiB` (взвешенный семафор по оценке `пиксели·8 + 6.5 MiB`; таймаутнувшая попытка держит свою долю до реального завершения декодера), время `JOB_TIMEOUT_SECONDS`, контейнер `768M`/`2 CPU`, `GOMEMLIMIT=600MiB`, `prefetch=concurrency`.
|
||||
|
||||
**Staging cleanup**: удаляется сразу после COMMIT `ready`; у `failed` хранится `STAGING_RETENTION_SECONDS=86400` (для replay), потом sweeper удаляет; брошенные `pending` — через 15 мин. Текущую работу (`queued/processing`) sweeper не выбирает.
|
||||
|
||||
**API-переход 4 → 5**: upload `201 ready` → `202 queued`; `GET /attachments/{id}` показывает `queued/processing/ready/failed`, `variants` только у ready, `error_code` только у failed; `content` (любой variant) для не-ready → 409, отсутствующий variant у ready → 404, неизвестный → 400; к сообщению можно прикрепить queued/processing/ready, failed → 409.
|
||||
|
||||
**Что перенесено из lab4 и что изменено.** Перенесены без изменений пользователи, сессии (Argon2id, Valkey-кэш), чаты, сообщения, идемпотентность, keyset-пагинация, `X-Instance-Id`, HAProxy, S3 (SeaweedFS), privacy bucket, хуки restart/stop-one. Изменено: upload (staging + outbox вместо прямого PUT в конечный ключ), из API убрано полное декодирование, `restart.*` теперь перезапускает и RabbitMQ/worker. **Удалён reconciler lab4** (трек 2 lab4): он считал «осиротевшими» все объекты вне `attachments/`, т.е. удалил бы staging и варианты; его функции по staging/pending взял sweeper worker. Сверка S3↔БД для вариантов в этой работе не реализована (см. ограничения).
|
||||
|
||||
**Соответствие `.env.example`**: `BROKER_URL_FILE`, `WORKER_CONCURRENCY` (= prefetch), `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS`, `S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` используются как есть; добавлены `MAX_ACTIVE_JOBS`, `JOB_LEASE_SECONDS`, `JOB_BACKOFF_BASE/MAX`, `WORKER_DECODE_BUDGET_BYTES`, `WORKER_MEMORY/CPUS/GOMEMLIMIT`, `PENDING_UPLOAD_GRACE`, `FAULT_INJECTION`. `S3_ENDPOINT` в примере исправлен на `http://s3:8333` (SeaweedFS).
|
||||
|
||||
| Роль | Service | Образ | Health | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | lb | haproxy:3.0.5-alpine | `:8404/healthz` | — | app; 127.0.0.1:8080 | — |
|
||||
| API ×2 | api | mtusi-messenger:lab5 | `/health/ready` (только PostgreSQL) | нет (read-only rootfs) | app, data | database_url, cache_url, broker_url, s3_* |
|
||||
| Worker | worker | mtusi-messenger:lab5 (`worker`) | — (restart unless-stopped) | нет (read-only rootfs) | data | database_url, broker_url, s3_* |
|
||||
| Broker | rabbitmq | rabbitmq:4.0.5-management-alpine | `check_port_connectivity` | mqdata | data, не публикуется | rabbitmq_auth.conf |
|
||||
| БД | postgres | postgres:17.2-alpine3.21 | pg_isready | pgdata | data | pg_password |
|
||||
| Сессии | valkey | valkey/valkey:8.0.2-alpine3.21 | ping | — | data | valkey_auth |
|
||||
| S3 | s3 | chrislusf/seaweedfs:3.80 | `:8333/healthz` | s3data | data | s3_identities.json |
|
||||
| Bootstrap | migrate, s3-init | mtusi-messenger:lab5 | one-shot | — | data | database_url / s3_admin_* |
|
||||
|
||||
`/health/ready` зависит только от PostgreSQL: недоступность брокера/worker/S3 не снимает API с балансировки (текстовый мессенджер работает, upload при живом S3 продолжает отдавать 202 через outbox).
|
||||
|
||||
**Учебные триггеры сбоев** (`FAULT_INJECTION=1`, срабатывают один раз на файл по префиксу имени): `fault-api-after-commit` (API `os.Exit` после COMMIT до publish/ответа), `fault-worker-after-effect` (после записи S3 до COMMIT), `fault-worker-before-ack` (после COMMIT до ack), `fault-transient` (попытки 1–2 падают), `fault-flaky` (падают все попытки до replay), `fault-hang` (попытка 1 «замерзает» на 3×lease без heartbeat).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Окружение: Windows 11, Docker Desktop 29.7.2 (WSL2); лимиты: API 1 CPU/512M ×2, worker 2 CPU/768M (`GOMEMLIMIT=600MiB`), RabbitMQ 1 CPU/512M, S3 1 CPU/1G, PostgreSQL 2 CPU/1G. Дата прогона — 2026-11-11/12.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Уровень 3
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke (202 → ready, оригинал, JPEG thumbnail 256×160) | `make test` | `PASS: lab5 public API scenario` | `evidence/smoke.txt` |
|
||||
| Минимум lab4 сохранён, сохранность после рестарта всех хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` (stop api/worker/lb, restart postgres/valkey/s3/rabbitmq) | `PASS: persistence preserved confirmed data and existing session` | `evidence/persistence.txt` |
|
||||
| Отказ одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | 117 записей, 0 ошибок, восстановление 3.97 с, `PASS` | `evidence/failover.txt` |
|
||||
| 202 только после устойчивого приёма; остановка API после ответа не теряет работу | `lab5ops.py worker-down`: worker остановлен → upload → `restart api` + `restart rabbitmq` → `start worker` | `202 queued`; сообщение с queued-вложением 201; content 409; текстовые сообщения 201; в очереди 1 сообщение до и **после** рестарта API+broker; после старта worker ready за 0.34 с | `evidence/worker-down.json` |
|
||||
| Отдельные broker и worker, durable + at-least-once, ack после эффекта | `compose.yaml` (rabbitmq quorum, worker), сценарии `effect-crash`/`ack-crash` | Сообщение пережило рестарт брокера с volume; повтор доставки после убийства worker завершил ту же job | `evidence/worker-down.json`, `evidence/effect-crash.json` |
|
||||
| Недоступность worker не вызывает синхронный resize | API не декодирует изображения вообще (`inspectImage` читает только заголовок) | upload при остановленном worker 202, статус `queued` до запуска worker | `evidence/worker-down.json` |
|
||||
| Состояния queued/processing/ready/failed; thumbnail по контракту; original неизменен | `worker-down`, `invalid`, `replay` | seen `processing→ready`, `failed (invalid_image / retry_exhausted)`; thumbnail реальный JPEG 256×160; sha256 оригинала совпадает | `evidence/*.json` (`ready_check`) |
|
||||
| Ограниченный retry + финальный failed с безопасным кодом | `lab5ops.py replay` (`JOB_MAX_ATTEMPTS=5`) | 5 попыток → `failed`, `error_code=retry_exhausted`, без stack trace | `evidence/replay.json` |
|
||||
| Повтор одной job не создаёт второго набора | `effect-crash`, `ack-crash`, `hang` | везде `variant_rows=3`, в S3 `variants/<chat>/<att>/` ровно 3 объекта | `evidence/effect-crash.json`, `ack-crash.json`, `hang.json` |
|
||||
|
||||
### Уровень 4
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| crop 128×128 и optimized WebP; параметры описаны | любой ready (`worker-down`), `make unit` | `thumbnail image/jpeg 256x160`, `crop image/jpeg 128x128`, `optimized image/webp 320x200`; unit-тесты: Fit, EXIF orientation 6 (320×200 → thumbnail 160×256), alpha → белый | `evidence/worker-down.json`, `src/internal/imaging/imaging_test.go` |
|
||||
| Невалидные/огромные изображения ограничены по bytes, пикселям, времени, памяти | `lab5ops.py invalid`, `lab5ops.py limits` | битый PNG с верным заголовком: 202 → `failed invalid_image` за **1** попытку (постоянная ошибка не ретраится); API: pixel bomb 20000×20000 → 413, не-изображение → 415, >10 MiB → 413, `../` в имени → 400; 6000×6000 (36 Мпикс) → ready за 5.2 с, пик памяти worker 491 MiB при лимите 768 MiB | `evidence/invalid.json`, `evidence/limits.json` |
|
||||
| Разрыв PostgreSQL ↔ broker закрыт outbox; crash между COMMIT и publish | `lab5ops.py api-crash` (`fault-api-after-commit`) | клиент получил 502 (API-процесс умер после COMMIT), API-контейнер перезапущен (`RestartCount 0→1`), строку outbox опубликовал другой relay, job `ready` без ручных действий; события `queued → started → ready` | `evidence/api-crash.json` |
|
||||
| Backoff с jitter, постоянные vs временные, DLQ/реестр failed, безопасный replay | `lab5ops.py retry`, `replay`, `dlq` | паузы перед попытками 2 и 3: 0.94 с и 1.81 с (диапазоны [0.5,1] и [1,2]); `fault-flaky`: 5 попыток → `failed retry_exhausted`; content 409, прикрепление к сообщению 409; `replay` → `{"result":"replayed","attempt":6}` → ready; повторный replay → `noop: job already ready`; malformed сообщение → `attachment.process.dlq` 0→1 | `evidence/retry.json`, `replay.json`, `dlq.json` |
|
||||
| Остановка worker после записи объекта, до ack — повтор завершает тот же результат | `lab5ops.py effect-crash` (после S3 до COMMIT), `ack-crash` (после COMMIT до ack) | effect-crash: события `started → taken_over → ready`, worker `RestartCount 0→1`, 3 варианта; ack-crash: `ready` один раз, после рестарта redelivery → лог `job delivery is a no-op result=done redelivered=true` | `evidence/effect-crash.json`, `evidence/ack-crash.json` |
|
||||
| Timeout/lease для зависшего processing | `lab5ops.py hang` | попытка 1 зависла без heartbeat → через 31 с sweeper: `retry_scheduled lease_expired` → попытка 2 → ready; «оживший» старый исполнитель: `stale attempt finished after takeover; result already owned by newer attempt`, вариантов по-прежнему 3 | `evidence/hang.json` |
|
||||
| Ограниченная конкуренция/prefetch | `WORKER_CONCURRENCY` = число goroutine = `basic.qos prefetch`; бюджет декодирования; `MAX_ACTIVE_JOBS` | см. трек 1: при `MAX_ACTIVE_JOBS=20` принято ровно 20 из 80 | `evidence/track1/*.json` |
|
||||
| Очистка staging после безопасной фиксации | все сценарии ready | `staging_removed=true`, в S3 `staging/<id>` 0 объектов, конечный оригинал 1; у failed staging сохраняется для replay | `evidence/worker-down.json` (`s3`) |
|
||||
|
||||
### Трек 1 — очередь под перегрузкой
|
||||
|
||||
**Гипотеза.** Pipeline изображений CPU-bound: увеличение `WORKER_CONCURRENCY` (= prefetch) повышает throughput почти линейно, пока число параллельных задач не упрётся в CPU-лимит worker (2 CPU), а для больших изображений — в бюджет памяти декодирования (384 MiB ≈ 2 задачи по 20 Мпикс). Дальше рост concurrency не даёт throughput, но увеличивает RSS. Без ограничения принятой работы очередь и время до ready растут неограниченно; с `MAX_ACTIVE_JOBS` лишняя работа отклоняется сразу (503 + `Retry-After`), а принятая гарантированно дренируется.
|
||||
|
||||
**Методика** (`scripts/track1/run_track1.py`, `make track1`). Для каждой настройки worker пересоздаётся с `WORKER_CONCURRENCY ∈ {1,2,4}` (те же лимиты 2 CPU / 768M / `GOMEMLIMIT=600MiB`, бюджет декодирования 384 MiB), для каждого размера — новый чат и **открытый burst** N загрузок 8 клиентскими потоками без ожидания обработки:
|
||||
- S = `tests/fixtures/sample.png` 320×200 (112 КБ), N=120;
|
||||
- M = 2000×1500 PNG с шумовыми строками (1.17 МБ), N=40;
|
||||
- L = 5000×4000 PNG-градиент (66 КБ на диске, 20 Мпикс после декодирования), N=16.
|
||||
|
||||
Метрики: время до ready каждой job = `finished_at − created_at` строки `attachment_jobs` (часы сервера, без клиентского polling); throughput = ready / (последний finished − первый created); queue lag — выборка каждые 0.5 с: активные jobs (queued+processing), `messages_ready` в RabbitMQ (`rabbitmqctl list_queues`), возраст самой старой незавершённой job; RSS — пик `docker stats` контейнера worker. Прогрев: стенд уже работал, по одному прогону на ячейку (ограничение — см. ниже). Raw: `evidence/track1/matrix.json` (с таймлайном lag), `evidence/track1/backpressure.json`.
|
||||
|
||||
| Размер | conc/prefetch | принято / ready / failed | throughput, job/s | p50 / p95 до ready, с | max queue (DB / MQ ready) | max возраст старейшей job, с | пик RSS worker, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| S 320×200 | 1 | 120 / 120 / 0 | 9.38 | 6.3 / 11.3 | 51 / 112 | 10.9 | 59 |
|
||||
| S 320×200 | 2 | 120 / 120 / 0 | 16.25 | 3.6 / 6.2 | 63 / 64 | 6.4 | 23 |
|
||||
| S 320×200 | 4 | 120 / 120 / 0 | **23.27** | **2.3 / 3.9** | 61 / 32 | 0.5 | 29 |
|
||||
| M 2000×1500 | 1 | 40 / 40 / 0 | 0.56 | 36.8 / 68.1 | 36 / 38 | 69.3 | 165 |
|
||||
| M 2000×1500 | 2 | 40 / 40 / 0 | **1.11** | **19.2 / 35.0** | 36 / 34 | 33.8 | 328 |
|
||||
| M 2000×1500 | 4 | 40 / 40 / 0 | 1.09 | 21.1 / 35.8 | 36 / 36 | 32.1 | 513 |
|
||||
| L 5000×4000 | 1 | 16 / 16 / 0 | 0.37 | 24.2 / 42.8 | 16 / 14 | 38.3 | 601 |
|
||||
| L 5000×4000 | 2 | 16 / 16 / 0 | **0.72** | **13.6 / 22.0** | 16 / 14 | 21.5 | 585 |
|
||||
| L 5000×4000 | 4 | 16 / 16 / 0 | 0.73 | 13.7 / 21.7 | 16 / 12 | 19.6 | 593 |
|
||||
|
||||
**Backpressure и восстановление** (`backpressure`): API с `MAX_ACTIVE_JOBS=20`, worker concurrency 1, burst 80 M-изображений за 1.1 с → **принято ровно 20, 60 × 503 `unavailable` с `Retry-After: 5`**, ни одного другого кода; максимум активных jobs по выборкам 18 ≤ 20. Все 20 принятых дошли до ready (0 failed, 0 retry), очередь опустела за 37.5 с (throughput 0.56 job/s — как в матрице для conc=1), и **сразу после перегрузки** новый upload стал ready за 0.34 с — система вернулась к idle-латентности без ручных действий. Первая версия ограничения (только предпроверка до чтения тела) пропустила 25 из 80 при пределе 20 (гонка check-then-insert между параллельными загрузками) — после этого добавлена повторная проверка в транзакции под `pg_advisory_xact_lock`, граница стала строгой.
|
||||
|
||||
**Выводы.**
|
||||
1. Для мелких изображений накладные расходы (БД-транзакции, S3 PUT ×4, ack) доминируют, а не CPU: throughput растёт 9.4 → 16.3 → 23.3 job/s (×2.5 при ×4 concurrency), p95 падает 11.3 → 3.9 с, RSS остаётся < 60 MiB. Здесь concurrency 4 однозначно лучше.
|
||||
2. Для M и L рост останавливается на concurrency 2 (= 2 CPU лимита): 1→2 даёт ×2 throughput и ×2 меньше p95, 2→4 — 0 % (1.11 → 1.09 и 0.72 → 0.73), зато RSS для M растёт 328 → 513 MiB. Для L concurrency 4 фактически не включается: бюджет декодирования 384 MiB пропускает только 2 задачи по 20 Мпикс одновременно, поэтому RSS держится ~590 MiB (у потолка `GOMEMLIMIT`), а OOM нет ни в одном прогоне. Гипотеза подтверждена: оптимум concurrency ≈ число CPU для тяжёлых изображений, больше — только трата памяти.
|
||||
3. Компромисс latency/ресурсы: при фиксированном CPU latency в очереди определяется законом Литтла — время до ready ≈ длина очереди / throughput; при неограниченном приёме p95 растёт линейно с burst (M, conc=1: 68 с для 40 файлов). Ограничение принятой работы делает худшую задержку предсказуемой (≈ `MAX_ACTIVE_JOBS` / throughput: 20 / 0.56 ≈ 36 с, наблюдали max 35 с) ценой явного 503 для клиента.
|
||||
4. Выбранная конфигурация по умолчанию: `WORKER_CONCURRENCY=2` (=CPU), бюджет 384 MiB как предохранитель памяти, `MAX_ACTIVE_JOBS=500` (для демонстрации backpressure — 20).
|
||||
|
||||
**Ограничения измерения.** По одному прогону на ячейку (разброс не оценён, повтор — `make track1`); один хост Windows/WSL2, клиент и стенд делят CPU; RSS — выборки `docker stats` раз в ~1–2 с, кратковременные пики могут быть пропущены; L-изображение синтетическое и хорошо сжимается (нагрузка — декодирование/ресайз, а не передача байтов); размер M-файлов искусственный (шум каждые 8 строк). Выводы относятся к этим лимитам и профилю, а не к production.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| worker остановлен, upload, рестарт API и RabbitMQ (volumes сохранены) | 202, задача не теряется | 202 queued; сообщение в quorum-очереди до и после рестарта; ready через 0.34 с после старта worker | потерь нет |
|
||||
| API умирает между COMMIT и publish | outbox опубликует | клиент 502, job ready через ~0.2 с (relay другой реплики/worker) | потерь нет; клиент не знает id — повтор upload создаст второе вложение (контракт это допускает) |
|
||||
| worker умирает после записи S3 до COMMIT | redelivery, перехват lease, тот же результат | `taken_over → ready`, 3 варианта | ~1 с (рестарт контейнера), дублей нет |
|
||||
| worker умирает после COMMIT до ack | redelivery — no-op | `result=done redelivered=true`, ack | дублей нет |
|
||||
| worker «завис» | lease истекает, новая попытка | ready через 31 с (lease 30 с) | старая попытка не перезаписала результат |
|
||||
| временные ошибки | backoff с jitter | 0.94 с, 1.81 с | ready на 3-й попытке |
|
||||
| постоянная ошибка (битый PNG) | failed без ретраев | `invalid_image` за 1 попытку | — |
|
||||
| malformed сообщение в брокере | DLQ | DLQ 0→1 | — |
|
||||
| перегрузка (80 загрузок при лимите 20) | 503 + Retry-After, работа ограничена | 20 принято, 60 × 503 `Retry-After: 5`, после дренажа новый файл ready за 0.34 с | принятые 20 все ready |
|
||||
| рестарт всех хранилищ (`restart.cmd`) | данные, сессия, файл сохранены | `make test-persistence` PASS | — |
|
||||
| остановка одной API-реплики | запись продолжается | 0 ошибок, 3.97 с до первой записи после hook | — |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-host: RabbitMQ — одна нода (quorum-очередь из одного члена), PostgreSQL и S3 — по одному экземпляру. Выдерживаются рестарты процессов с сохранёнными volumes, а не потеря диска/хоста.
|
||||
- At-least-once + идемпотентный эффект ≠ exactly-once: один и тот же S3-ключ может быть записан несколько раз (одинаковыми байтами), а дубль сообщения — получен; «ровно один раз» гарантируется только для зафиксированного результата в PostgreSQL.
|
||||
- При crash между COMMIT и ответом клиент не узнаёт `attachment_id`; без расширения контракта (Idempotency-Key для upload) повтор создаст второе вложение.
|
||||
- Таймаут не прерывает декодер Go насильно: зависшая попытка освобождает свою долю бюджета памяти только по фактическому завершению декодирования; защиту от бесконечного декодера даёт лимит контейнера.
|
||||
- Сверка S3 ↔ БД для вариантов (reconciler lab4) не перенесена; объекты от устаревших попыток всегда совпадают с зафиксированными по ключу, но «сиротские» объекты удалённых вложений не собираются.
|
||||
- Fault injection по имени файла — учебный механизм, выключается `FAULT_INJECTION=0`.
|
||||
- Worker использует ту же S3-идентичность, что и API; следующий шаг — отдельная идентичность с правами только на нужные prefix-ы.
|
||||
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_STORE: ${SESSION_STORE:-valkey}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT: ${S3_TIMEOUT:-15s}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8}
|
||||
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-60s}
|
||||
GOMEMLIMIT: ${API_GOMEMLIMIT:-400MiB}
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
MAX_ACTIVE_JOBS: ${MAX_ACTIVE_JOBS:-500}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-0}
|
||||
|
||||
x-worker-env: &worker-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
DB_MAX_CONNS: "${WORKER_DB_MAX_CONNS:-8}"
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_DECODE_BUDGET_BYTES: ${WORKER_DECODE_BUDGET_BYTES:-402653184}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
|
||||
JOB_BACKOFF_BASE: ${JOB_BACKOFF_BASE:-1s}
|
||||
JOB_BACKOFF_MAX: ${JOB_BACKOFF_MAX:-60s}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
PENDING_UPLOAD_GRACE: ${PENDING_UPLOAD_GRACE:-15m}
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-0}
|
||||
GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-600MiB}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.2-alpine3.21
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
|
||||
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
|
||||
secrets: [pg_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [data]
|
||||
shm_size: 256mb
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1G}"}
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.2-alpine3.21
|
||||
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
|
||||
volumes:
|
||||
- ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro
|
||||
secrets: [valkey_auth]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.80
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -s3
|
||||
- -s3.port=8333
|
||||
- -s3.config=/run/secrets/s3_identities
|
||||
- -master.volumeSizeLimitMB=64
|
||||
- -volume.max=100
|
||||
secrets: [s3_identities]
|
||||
volumes:
|
||||
- s3data:/data
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
start_period: 60s
|
||||
retries: 60
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: "${S3_MEMORY:-1G}"}
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never
|
||||
command: ["s3-init"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets: [s3_admin_access_key_id, s3_admin_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 64M}
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_BUILD: ${APP_BUILD:-dev}
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never
|
||||
environment: *api-env
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=1m
|
||||
secrets: [database_url, cache_url, broker_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [app, data]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
start_period: 5s
|
||||
retries: 3
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"}
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.0.5-management-alpine
|
||||
hostname: rabbitmq
|
||||
secrets:
|
||||
- source: rabbitmq_auth
|
||||
target: /etc/rabbitmq/conf.d/20-auth.conf
|
||||
volumes:
|
||||
- mqdata:/var/lib/rabbitmq
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
start_period: 20s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never
|
||||
command: ["worker"]
|
||||
environment: *worker-env
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=1m
|
||||
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-768M}"}
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
- "[::1]:${APP_PORT:-8080}:8080"
|
||||
networks: [app]
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128M}
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
app: {}
|
||||
data: {}
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
s3data: {}
|
||||
mqdata: {}
|
||||
|
||||
secrets:
|
||||
pg_password:
|
||||
file: ./secrets/pg_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
valkey_auth:
|
||||
file: ./secrets/valkey_auth.conf
|
||||
s3_identities:
|
||||
file: ./secrets/s3_identities.json
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
rabbitmq_auth:
|
||||
file: ./secrets/rabbitmq_auth.conf
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"scenario": "ack-crash",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:24:36Z",
|
||||
"fault": "worker os.Exit after result COMMIT + staging cleanup, before ack",
|
||||
"attachment_id": "fc38c3d8-b1b7-440f-b6c5-9b315979731b",
|
||||
"upload": 202,
|
||||
"worker_restart_counts_before": {
|
||||
"d3259669fad0": 1
|
||||
},
|
||||
"worker_restart_counts_after": {
|
||||
"d3259669fad0": 2
|
||||
},
|
||||
"statuses_seen": [
|
||||
"processing",
|
||||
"ready"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:24:38.431"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:24:38.460"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "ready",
|
||||
"error_code": "",
|
||||
"at": "23:24:38.529"
|
||||
}
|
||||
],
|
||||
"job_after": {
|
||||
"job_id": "4b9eb573-b6b4-4f6c-bec9-0d73147bab15",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"s3": {
|
||||
"variants_objects": 3,
|
||||
"final_original": 1,
|
||||
"staging_objects": 0
|
||||
},
|
||||
"redelivery_noop_log": [
|
||||
{
|
||||
"result": "done",
|
||||
"redelivered": true
|
||||
}
|
||||
],
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"scenario": "api-crash",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:24:10Z",
|
||||
"fault": "API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response",
|
||||
"attachment_id": "4e18930b-bd0f-4db5-a033-35fbd74c93a1",
|
||||
"client_view": "client saw failure: -a15d-9d5fe0759a8c/attachments: expected [202], received 502",
|
||||
"job_right_after_crash": {
|
||||
"job_id": "8dadbdde-897c-4acb-ae12-b7565b03dbb4",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"api_restart_counts_before": {
|
||||
"78c8984ff002": 0,
|
||||
"4c38672b42d2": 0
|
||||
},
|
||||
"api_restart_counts_after": {
|
||||
"78c8984ff002": 1,
|
||||
"4c38672b42d2": 0
|
||||
},
|
||||
"statuses_seen": [
|
||||
"ready"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:24:12.312"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:24:12.463"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "ready",
|
||||
"error_code": "",
|
||||
"at": "23:24:12.538"
|
||||
}
|
||||
],
|
||||
"job_after": {
|
||||
"job_id": "8dadbdde-897c-4acb-ae12-b7565b03dbb4",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"scenario": "dlq",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:25:29Z",
|
||||
"published": {
|
||||
"routed": true
|
||||
},
|
||||
"dlq_before": {
|
||||
"ready": 0,
|
||||
"unacked": 0,
|
||||
"consumers": 0
|
||||
},
|
||||
"dlq_after": {
|
||||
"ready": 1,
|
||||
"unacked": 0,
|
||||
"consumers": 0
|
||||
},
|
||||
"worker_log": [
|
||||
"malformed job message -> DLQ"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"scenario": "effect-crash",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:24:19Z",
|
||||
"fault": "worker os.Exit after writing original+variants to S3, before result COMMIT and ack",
|
||||
"attachment_id": "4e2f8752-7072-4dc0-ae24-eccb7f6d25ff",
|
||||
"upload": 202,
|
||||
"worker_restart_counts_before": {
|
||||
"d3259669fad0": 0
|
||||
},
|
||||
"worker_restart_counts_after": {
|
||||
"d3259669fad0": 1
|
||||
},
|
||||
"statuses_seen": [
|
||||
"queued",
|
||||
"processing",
|
||||
"ready"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:24:21.431"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:24:21.482"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "taken_over",
|
||||
"error_code": "",
|
||||
"at": "23:24:22.154"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "ready",
|
||||
"error_code": "",
|
||||
"at": "23:24:22.573"
|
||||
}
|
||||
],
|
||||
"job_after": {
|
||||
"job_id": "f945f7fb-dd8f-461b-b3e5-7cae6db38fcc",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"s3": {
|
||||
"variants_objects": 3,
|
||||
"final_original": 1,
|
||||
"staging_objects": 0
|
||||
},
|
||||
"redelivery_noop_log": [],
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 117, "failed_attempts": 0, "first_recovery_seconds": 3.972}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"scenario": "hang",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:25:57Z",
|
||||
"fault": "attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)",
|
||||
"attachment_id": "9eb5e502-1491-4ae3-93ae-5031997a921d",
|
||||
"seconds_to_ready": 31.6,
|
||||
"statuses_seen": [
|
||||
"queued",
|
||||
"processing",
|
||||
"queued",
|
||||
"ready"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:25:57.874"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:25:57.929"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "lease_expired",
|
||||
"at": "23:26:29.068"
|
||||
},
|
||||
{
|
||||
"attempt": "2",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:26:29.319"
|
||||
},
|
||||
{
|
||||
"attempt": "2",
|
||||
"event": "ready",
|
||||
"error_code": "",
|
||||
"at": "23:26:29.391"
|
||||
}
|
||||
],
|
||||
"job_after": {
|
||||
"job_id": "f955fe0d-3b17-42bf-8df8-aae785e1ae68",
|
||||
"status": "ready",
|
||||
"attempt": 2,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 2,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"s3": {
|
||||
"variants_objects": 3,
|
||||
"final_original": 1,
|
||||
"staging_objects": 0
|
||||
},
|
||||
"stale_attempt_logs": [
|
||||
"stale attempt finished after takeover; result already owned by newer attempt"
|
||||
],
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"scenario": "invalid",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:25:26Z",
|
||||
"corrupt_png": {
|
||||
"upload": 202,
|
||||
"statuses_seen": [
|
||||
"processing",
|
||||
"failed"
|
||||
],
|
||||
"error_code": "invalid_image",
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:25:26.980"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:25:27.016"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "failed",
|
||||
"error_code": "invalid_image",
|
||||
"at": "23:25:27.027"
|
||||
}
|
||||
],
|
||||
"job": {
|
||||
"job_id": "7ead7c8c-0dd7-4b4a-8c2a-807323b1673a",
|
||||
"status": "failed",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "invalid_image",
|
||||
"variant_rows": 0,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": false
|
||||
}
|
||||
},
|
||||
"api_rejects_before_queue": {
|
||||
"pixel_bomb_20000x20000": 413,
|
||||
"not_an_image": 415,
|
||||
"over_10MiB": 413,
|
||||
"path_in_filename": 400
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"scenario": "limits",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:25:46Z",
|
||||
"image": "6000x6000 PNG (36 MPx, 116950 bytes)",
|
||||
"seconds_to_ready": 5.2,
|
||||
"worker_peak_mem_mib": 490.6,
|
||||
"worker_mem_limit_mib": 768,
|
||||
"variants": {
|
||||
"thumbnail": "256x256",
|
||||
"crop": "128x128",
|
||||
"optimized": "1280x1280"
|
||||
},
|
||||
"statuses_seen": [
|
||||
"processing",
|
||||
"ready"
|
||||
],
|
||||
"job": {
|
||||
"job_id": "a0763969-5958-4662-b71a-5e861560a79b",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,154 @@
|
||||
{
|
||||
"scenario": "replay",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:28:58Z",
|
||||
"fault": "transient error on every attempt <= JOB_MAX_ATTEMPTS",
|
||||
"attachment_id": "adcb5aa2-0565-4bc7-b88c-c44cab442533",
|
||||
"statuses_seen": [
|
||||
"queued",
|
||||
"processing",
|
||||
"queued",
|
||||
"failed"
|
||||
],
|
||||
"failed_metadata": {
|
||||
"status": "failed",
|
||||
"error_code": "retry_exhausted",
|
||||
"variants": []
|
||||
},
|
||||
"content_when_failed": 409,
|
||||
"attach_failed_to_message": 409,
|
||||
"job_failed": {
|
||||
"job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d",
|
||||
"status": "failed",
|
||||
"attempt": 5,
|
||||
"max_attempts": 5,
|
||||
"error_code": "retry_exhausted",
|
||||
"variant_rows": 0,
|
||||
"outbox_rows": 5,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": false
|
||||
},
|
||||
"dlq_registry_query": "1",
|
||||
"replay_1": {
|
||||
"attempt": 6,
|
||||
"job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d",
|
||||
"result": "replayed"
|
||||
},
|
||||
"replay_2_after_ready": {
|
||||
"result": "noop",
|
||||
"reason": "job already ready; replay refused"
|
||||
},
|
||||
"statuses_after_replay": [
|
||||
"processing",
|
||||
"ready"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:28:59.257"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:28:59.318"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "injected_transient",
|
||||
"at": "23:28:59.324"
|
||||
},
|
||||
{
|
||||
"attempt": "2",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:29:00.268"
|
||||
},
|
||||
{
|
||||
"attempt": "2",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "injected_transient",
|
||||
"at": "23:29:00.277"
|
||||
},
|
||||
{
|
||||
"attempt": "3",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:29:02.077"
|
||||
},
|
||||
{
|
||||
"attempt": "3",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "injected_transient",
|
||||
"at": "23:29:02.085"
|
||||
},
|
||||
{
|
||||
"attempt": "4",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:29:05.701"
|
||||
},
|
||||
{
|
||||
"attempt": "4",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "injected_transient",
|
||||
"at": "23:29:05.708"
|
||||
},
|
||||
{
|
||||
"attempt": "5",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:29:11.048"
|
||||
},
|
||||
{
|
||||
"attempt": "5",
|
||||
"event": "failed",
|
||||
"error_code": "retry_exhausted",
|
||||
"at": "23:29:11.060"
|
||||
},
|
||||
{
|
||||
"attempt": "6",
|
||||
"event": "replay",
|
||||
"error_code": "",
|
||||
"at": "23:29:13.048"
|
||||
},
|
||||
{
|
||||
"attempt": "6",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:29:13.264"
|
||||
},
|
||||
{
|
||||
"attempt": "6",
|
||||
"event": "ready",
|
||||
"error_code": "",
|
||||
"at": "23:29:13.784"
|
||||
}
|
||||
],
|
||||
"job_after": {
|
||||
"job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d",
|
||||
"status": "ready",
|
||||
"attempt": 6,
|
||||
"max_attempts": 10,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 6,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"scenario": "retry",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:24:52Z",
|
||||
"fault": "attempts 1 and 2 raise a transient error (injected_transient)",
|
||||
"attachment_id": "1eadb246-c3b9-4152-b21e-1b5ec2de9033",
|
||||
"statuses_seen": [
|
||||
"queued",
|
||||
"processing",
|
||||
"ready"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "queued",
|
||||
"error_code": "",
|
||||
"at": "23:24:53.308"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:24:53.381"
|
||||
},
|
||||
{
|
||||
"attempt": "1",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "injected_transient",
|
||||
"at": "23:24:53.390"
|
||||
},
|
||||
{
|
||||
"attempt": "2",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:24:54.330"
|
||||
},
|
||||
{
|
||||
"attempt": "2",
|
||||
"event": "retry_scheduled",
|
||||
"error_code": "injected_transient",
|
||||
"at": "23:24:54.339"
|
||||
},
|
||||
{
|
||||
"attempt": "3",
|
||||
"event": "started",
|
||||
"error_code": "",
|
||||
"at": "23:24:56.146"
|
||||
},
|
||||
{
|
||||
"attempt": "3",
|
||||
"event": "ready",
|
||||
"error_code": "",
|
||||
"at": "23:24:56.565"
|
||||
}
|
||||
],
|
||||
"retry_gaps_seconds": [
|
||||
0.94,
|
||||
1.81
|
||||
],
|
||||
"expected_gap_ranges": [
|
||||
"attempt1: [0.5, 1.0] s",
|
||||
"attempt2: [1.0, 2.0] s"
|
||||
],
|
||||
"job_after": {
|
||||
"job_id": "09973bd1-2b97-43c2-9292-d3519960a2c7",
|
||||
"status": "ready",
|
||||
"attempt": 3,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 3,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,106 @@
|
||||
{
|
||||
"experiment": "backpressure",
|
||||
"started_at": "2026-11-12T07:56:12Z",
|
||||
"worker_limits": "cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB",
|
||||
"host": "Windows 11, Docker Desktop 29.7.2 (WSL2)",
|
||||
"max_active_jobs_bound": 20,
|
||||
"worker_concurrency": 1,
|
||||
"uploads": 80,
|
||||
"accepted_202": 20,
|
||||
"rejected_503": 60,
|
||||
"retry_after_header": [
|
||||
"5"
|
||||
],
|
||||
"other": [],
|
||||
"upload_burst_s": 1.22,
|
||||
"drained": true,
|
||||
"drain_s": 36.4,
|
||||
"max_active_observed": 18,
|
||||
"jobs": {
|
||||
"jobs": 21,
|
||||
"ready": 21,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 0.58,
|
||||
"pipeline_span_s": 36.38,
|
||||
"time_to_ready_s": {
|
||||
"p50": 17.728,
|
||||
"p95": 32.937,
|
||||
"max": 34.657
|
||||
}
|
||||
},
|
||||
"after_overload_upload_status": "ready",
|
||||
"after_overload_seconds_to_ready": 0.28,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 3.8,
|
||||
"active": 8,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 0.29
|
||||
},
|
||||
{
|
||||
"t": 7.7,
|
||||
"active": 18,
|
||||
"mq_ready": 18,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 4.38
|
||||
},
|
||||
{
|
||||
"t": 12.1,
|
||||
"active": 16,
|
||||
"mq_ready": 15,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 8.2
|
||||
},
|
||||
{
|
||||
"t": 16.0,
|
||||
"active": 14,
|
||||
"mq_ready": 12,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 12.58
|
||||
},
|
||||
{
|
||||
"t": 20.2,
|
||||
"active": 11,
|
||||
"mq_ready": 9,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 16.32
|
||||
},
|
||||
{
|
||||
"t": 23.8,
|
||||
"active": 9,
|
||||
"mq_ready": 9,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 20.4
|
||||
},
|
||||
{
|
||||
"t": 28.3,
|
||||
"active": 7,
|
||||
"mq_ready": 6,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 24.09
|
||||
},
|
||||
{
|
||||
"t": 31.5,
|
||||
"active": 4,
|
||||
"mq_ready": 3,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 28.48
|
||||
},
|
||||
{
|
||||
"t": 36.5,
|
||||
"active": 2,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 31.68
|
||||
},
|
||||
{
|
||||
"t": 40.0,
|
||||
"active": 0,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 0.0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,682 @@
|
||||
{
|
||||
"experiment": "matrix",
|
||||
"started_at": "2026-11-11T23:38:36Z",
|
||||
"worker_limits": "cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB",
|
||||
"host": "Windows 11, Docker Desktop 29.7.2 (WSL2)",
|
||||
"runs": [
|
||||
{
|
||||
"size": "S_320x200",
|
||||
"image_bytes": 111817,
|
||||
"worker_concurrency_prefetch": 1,
|
||||
"uploads": 120,
|
||||
"accepted_202": 120,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 1.09,
|
||||
"drained": true,
|
||||
"jobs": 120,
|
||||
"ready": 120,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 9.38,
|
||||
"pipeline_span_s": 12.79,
|
||||
"time_to_ready_s": {
|
||||
"p50": 6.325,
|
||||
"p95": 11.297,
|
||||
"max": 11.759
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 51,
|
||||
"max_mq_ready": 112,
|
||||
"max_oldest_job_age_s": 10.94
|
||||
},
|
||||
"worker_peak_rss_mib": 59.3,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 6.9,
|
||||
"active": 50,
|
||||
"mq_ready": 112,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 0.51
|
||||
},
|
||||
{
|
||||
"t": 11.0,
|
||||
"active": 51,
|
||||
"mq_ready": 64,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 7.09
|
||||
},
|
||||
{
|
||||
"t": 16.8,
|
||||
"active": 10,
|
||||
"mq_ready": 12,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 10.94
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "M_2000x1500",
|
||||
"image_bytes": 1165217,
|
||||
"worker_concurrency_prefetch": 1,
|
||||
"uploads": 40,
|
||||
"accepted_202": 40,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 1.23,
|
||||
"drained": true,
|
||||
"jobs": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 0.56,
|
||||
"pipeline_span_s": 70.84,
|
||||
"time_to_ready_s": {
|
||||
"p50": 36.797,
|
||||
"p95": 68.11,
|
||||
"max": 69.761
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 36,
|
||||
"max_mq_ready": 38,
|
||||
"max_oldest_job_age_s": 69.31
|
||||
},
|
||||
"worker_peak_rss_mib": 165.3,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 6.6,
|
||||
"active": 14,
|
||||
"mq_ready": 38,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 0.39
|
||||
},
|
||||
{
|
||||
"t": 10.5,
|
||||
"active": 36,
|
||||
"mq_ready": 36,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 7.22
|
||||
},
|
||||
{
|
||||
"t": 16.4,
|
||||
"active": 34,
|
||||
"mq_ready": 33,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 11.15
|
||||
},
|
||||
{
|
||||
"t": 20.7,
|
||||
"active": 31,
|
||||
"mq_ready": 30,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 16.91
|
||||
},
|
||||
{
|
||||
"t": 25.2,
|
||||
"active": 28,
|
||||
"mq_ready": 27,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 21.14
|
||||
},
|
||||
{
|
||||
"t": 30.4,
|
||||
"active": 26,
|
||||
"mq_ready": 24,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 25.53
|
||||
},
|
||||
{
|
||||
"t": 34.5,
|
||||
"active": 23,
|
||||
"mq_ready": 21,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 30.62
|
||||
},
|
||||
{
|
||||
"t": 40.8,
|
||||
"active": 21,
|
||||
"mq_ready": 19,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 34.71
|
||||
},
|
||||
{
|
||||
"t": 44.9,
|
||||
"active": 17,
|
||||
"mq_ready": 16,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 40.9
|
||||
},
|
||||
{
|
||||
"t": 51.7,
|
||||
"active": 15,
|
||||
"mq_ready": 13,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 44.93
|
||||
},
|
||||
{
|
||||
"t": 55.8,
|
||||
"active": 11,
|
||||
"mq_ready": 10,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 51.73
|
||||
},
|
||||
{
|
||||
"t": 61.0,
|
||||
"active": 9,
|
||||
"mq_ready": 7,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 55.64
|
||||
},
|
||||
{
|
||||
"t": 65.5,
|
||||
"active": 6,
|
||||
"mq_ready": 5,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 60.82
|
||||
},
|
||||
{
|
||||
"t": 69.7,
|
||||
"active": 3,
|
||||
"mq_ready": 2,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 65.23
|
||||
},
|
||||
{
|
||||
"t": 75.2,
|
||||
"active": 1,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 69.31
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "L_5000x4000",
|
||||
"image_bytes": 66065,
|
||||
"worker_concurrency_prefetch": 1,
|
||||
"uploads": 16,
|
||||
"accepted_202": 16,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 0.18,
|
||||
"drained": true,
|
||||
"jobs": 16,
|
||||
"ready": 16,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 0.37,
|
||||
"pipeline_span_s": 42.92,
|
||||
"time_to_ready_s": {
|
||||
"p50": 24.227,
|
||||
"p95": 42.775,
|
||||
"max": 42.775
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 16,
|
||||
"max_mq_ready": 14,
|
||||
"max_oldest_job_age_s": 38.34
|
||||
},
|
||||
"worker_peak_rss_mib": 601.1,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 3.4,
|
||||
"active": 16,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 0.4
|
||||
},
|
||||
{
|
||||
"t": 8.4,
|
||||
"active": 15,
|
||||
"mq_ready": 14,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 4.38
|
||||
},
|
||||
{
|
||||
"t": 13.3,
|
||||
"active": 13,
|
||||
"mq_ready": 12,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 9.25
|
||||
},
|
||||
{
|
||||
"t": 17.9,
|
||||
"active": 11,
|
||||
"mq_ready": 10,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 14.16
|
||||
},
|
||||
{
|
||||
"t": 23.4,
|
||||
"active": 10,
|
||||
"mq_ready": 8,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 18.7
|
||||
},
|
||||
{
|
||||
"t": 27.3,
|
||||
"active": 8,
|
||||
"mq_ready": 6,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 24.23
|
||||
},
|
||||
{
|
||||
"t": 33.6,
|
||||
"active": 6,
|
||||
"mq_ready": 5,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 28.08
|
||||
},
|
||||
{
|
||||
"t": 37.6,
|
||||
"active": 4,
|
||||
"mq_ready": 3,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 34.53
|
||||
},
|
||||
{
|
||||
"t": 44.6,
|
||||
"active": 2,
|
||||
"mq_ready": 1,
|
||||
"mq_unacked": 1,
|
||||
"oldest_age_s": 38.34
|
||||
},
|
||||
{
|
||||
"t": 48.5,
|
||||
"active": 0,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "S_320x200",
|
||||
"image_bytes": 111817,
|
||||
"worker_concurrency_prefetch": 2,
|
||||
"uploads": 120,
|
||||
"accepted_202": 120,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 1.05,
|
||||
"drained": true,
|
||||
"jobs": 120,
|
||||
"ready": 120,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 16.25,
|
||||
"pipeline_span_s": 7.39,
|
||||
"time_to_ready_s": {
|
||||
"p50": 3.649,
|
||||
"p95": 6.158,
|
||||
"max": 6.383
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 63,
|
||||
"max_mq_ready": 64,
|
||||
"max_oldest_job_age_s": 6.35
|
||||
},
|
||||
"worker_peak_rss_mib": 22.8,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 6.5,
|
||||
"active": 63,
|
||||
"mq_ready": 64,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 0.51
|
||||
},
|
||||
{
|
||||
"t": 10.6,
|
||||
"active": 1,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 6.35
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "M_2000x1500",
|
||||
"image_bytes": 1165247,
|
||||
"worker_concurrency_prefetch": 2,
|
||||
"uploads": 40,
|
||||
"accepted_202": 40,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 1.18,
|
||||
"drained": true,
|
||||
"jobs": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 1.11,
|
||||
"pipeline_span_s": 36.02,
|
||||
"time_to_ready_s": {
|
||||
"p50": 19.159,
|
||||
"p95": 35.018,
|
||||
"max": 35.103
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 36,
|
||||
"max_mq_ready": 34,
|
||||
"max_oldest_job_age_s": 33.77
|
||||
},
|
||||
"worker_peak_rss_mib": 328.2,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 3.6,
|
||||
"active": 11,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 0.27
|
||||
},
|
||||
{
|
||||
"t": 8.7,
|
||||
"active": 36,
|
||||
"mq_ready": 34,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 4.26
|
||||
},
|
||||
{
|
||||
"t": 13.5,
|
||||
"active": 30,
|
||||
"mq_ready": 28,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 9.11
|
||||
},
|
||||
{
|
||||
"t": 17.5,
|
||||
"active": 26,
|
||||
"mq_ready": 23,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 13.82
|
||||
},
|
||||
{
|
||||
"t": 23.2,
|
||||
"active": 20,
|
||||
"mq_ready": 17,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 17.73
|
||||
},
|
||||
{
|
||||
"t": 27.3,
|
||||
"active": 14,
|
||||
"mq_ready": 12,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 23.2
|
||||
},
|
||||
{
|
||||
"t": 33.9,
|
||||
"active": 10,
|
||||
"mq_ready": 6,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 27.16
|
||||
},
|
||||
{
|
||||
"t": 37.9,
|
||||
"active": 2,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 33.77
|
||||
},
|
||||
{
|
||||
"t": 44.8,
|
||||
"active": 0,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "L_5000x4000",
|
||||
"image_bytes": 66065,
|
||||
"worker_concurrency_prefetch": 2,
|
||||
"uploads": 16,
|
||||
"accepted_202": 16,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 0.18,
|
||||
"drained": true,
|
||||
"jobs": 16,
|
||||
"ready": 16,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 0.72,
|
||||
"pipeline_span_s": 22.17,
|
||||
"time_to_ready_s": {
|
||||
"p50": 13.631,
|
||||
"p95": 22.04,
|
||||
"max": 22.04
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 16,
|
||||
"max_mq_ready": 14,
|
||||
"max_oldest_job_age_s": 21.51
|
||||
},
|
||||
"worker_peak_rss_mib": 585.2,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 6.3,
|
||||
"active": 16,
|
||||
"mq_ready": 14,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 0.41
|
||||
},
|
||||
{
|
||||
"t": 10.4,
|
||||
"active": 12,
|
||||
"mq_ready": 10,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 7.17
|
||||
},
|
||||
{
|
||||
"t": 16.7,
|
||||
"active": 8,
|
||||
"mq_ready": 6,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 11.19
|
||||
},
|
||||
{
|
||||
"t": 20.7,
|
||||
"active": 4,
|
||||
"mq_ready": 2,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 17.55
|
||||
},
|
||||
{
|
||||
"t": 25.8,
|
||||
"active": 2,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 21.51
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "S_320x200",
|
||||
"image_bytes": 111817,
|
||||
"worker_concurrency_prefetch": 4,
|
||||
"uploads": 120,
|
||||
"accepted_202": 120,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 1.11,
|
||||
"drained": true,
|
||||
"jobs": 120,
|
||||
"ready": 120,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 23.27,
|
||||
"pipeline_span_s": 5.16,
|
||||
"time_to_ready_s": {
|
||||
"p50": 2.281,
|
||||
"p95": 3.945,
|
||||
"max": 4.099
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 61,
|
||||
"max_mq_ready": 32,
|
||||
"max_oldest_job_age_s": 0.51
|
||||
},
|
||||
"worker_peak_rss_mib": 29.0,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 4.9,
|
||||
"active": 61,
|
||||
"mq_ready": 32,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 0.51
|
||||
},
|
||||
{
|
||||
"t": 9.0,
|
||||
"active": 0,
|
||||
"mq_ready": 32,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 0.0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "M_2000x1500",
|
||||
"image_bytes": 1165242,
|
||||
"worker_concurrency_prefetch": 4,
|
||||
"uploads": 40,
|
||||
"accepted_202": 40,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 1.28,
|
||||
"drained": true,
|
||||
"jobs": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 1.09,
|
||||
"pipeline_span_s": 36.86,
|
||||
"time_to_ready_s": {
|
||||
"p50": 21.093,
|
||||
"p95": 35.792,
|
||||
"max": 35.83
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 36,
|
||||
"max_mq_ready": 36,
|
||||
"max_oldest_job_age_s": 32.06
|
||||
},
|
||||
"worker_peak_rss_mib": 513.1,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 5.5,
|
||||
"active": 9,
|
||||
"mq_ready": 36,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 0.28
|
||||
},
|
||||
{
|
||||
"t": 9.4,
|
||||
"active": 36,
|
||||
"mq_ready": 31,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 6.15
|
||||
},
|
||||
{
|
||||
"t": 13.9,
|
||||
"active": 32,
|
||||
"mq_ready": 24,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 9.87
|
||||
},
|
||||
{
|
||||
"t": 19.0,
|
||||
"active": 27,
|
||||
"mq_ready": 20,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 14.19
|
||||
},
|
||||
{
|
||||
"t": 22.8,
|
||||
"active": 20,
|
||||
"mq_ready": 20,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 19.04
|
||||
},
|
||||
{
|
||||
"t": 28.5,
|
||||
"active": 16,
|
||||
"mq_ready": 8,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 22.81
|
||||
},
|
||||
{
|
||||
"t": 32.3,
|
||||
"active": 12,
|
||||
"mq_ready": 8,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 28.43
|
||||
},
|
||||
{
|
||||
"t": 39.0,
|
||||
"active": 6,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 0,
|
||||
"oldest_age_s": 32.06
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"size": "L_5000x4000",
|
||||
"image_bytes": 66065,
|
||||
"worker_concurrency_prefetch": 4,
|
||||
"uploads": 16,
|
||||
"accepted_202": 16,
|
||||
"rejected": 0,
|
||||
"upload_burst_s": 0.18,
|
||||
"drained": true,
|
||||
"jobs": 16,
|
||||
"ready": 16,
|
||||
"failed": 0,
|
||||
"retried_jobs": 0,
|
||||
"throughput_jobs_per_s": 0.73,
|
||||
"pipeline_span_s": 21.84,
|
||||
"time_to_ready_s": {
|
||||
"p50": 13.692,
|
||||
"p95": 21.692,
|
||||
"max": 21.692
|
||||
},
|
||||
"queue_lag": {
|
||||
"max_active_jobs": 16,
|
||||
"max_mq_ready": 12,
|
||||
"max_oldest_job_age_s": 19.61
|
||||
},
|
||||
"worker_peak_rss_mib": 593.2,
|
||||
"lag_timeline": [
|
||||
{
|
||||
"t": 4.9,
|
||||
"active": 16,
|
||||
"mq_ready": 12,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 0.42
|
||||
},
|
||||
{
|
||||
"t": 8.9,
|
||||
"active": 12,
|
||||
"mq_ready": 8,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 5.73
|
||||
},
|
||||
{
|
||||
"t": 15.0,
|
||||
"active": 10,
|
||||
"mq_ready": 4,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 9.71
|
||||
},
|
||||
{
|
||||
"t": 18.9,
|
||||
"active": 6,
|
||||
"mq_ready": 2,
|
||||
"mq_unacked": 4,
|
||||
"oldest_age_s": 15.7
|
||||
},
|
||||
{
|
||||
"t": 25.9,
|
||||
"active": 2,
|
||||
"mq_ready": 0,
|
||||
"mq_unacked": 2,
|
||||
"oldest_age_s": 19.61
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"scenario": "worker-down",
|
||||
"result": "PASS",
|
||||
"started_at": "2026-11-11T23:23:18Z",
|
||||
"attachment_id": "4ebdeb92-a3d0-467d-b8cf-e7e7d45306c2",
|
||||
"message_id": "f0e4acb4-49b4-4c27-b726-911937e01552",
|
||||
"upload_status": 202,
|
||||
"upload_ms_worker_down": 99,
|
||||
"queue_before_restart": {
|
||||
"ready": 1,
|
||||
"unacked": 0,
|
||||
"consumers": 0
|
||||
},
|
||||
"job_before_restart": {
|
||||
"job_id": "a56aa984-d3eb-4716-a47d-f246d752426c",
|
||||
"status": "queued",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 0,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": false
|
||||
},
|
||||
"queue_after_api_and_broker_restart": {
|
||||
"ready": 1,
|
||||
"unacked": 0,
|
||||
"consumers": 0
|
||||
},
|
||||
"status_after_restart_before_worker": "queued",
|
||||
"seconds_to_ready_after_worker_start": 0.34,
|
||||
"statuses_seen": [
|
||||
"processing",
|
||||
"ready"
|
||||
],
|
||||
"steps": [
|
||||
"docker compose stop worker",
|
||||
"upload -> 202 queued",
|
||||
"message with queued attachment -> 201",
|
||||
"content while queued -> 409",
|
||||
"plain text message -> 201",
|
||||
"docker compose restart api rabbitmq (volumes kept)",
|
||||
"docker compose start worker"
|
||||
],
|
||||
"ready_check": {
|
||||
"variants": {
|
||||
"thumbnail": "image/jpeg 256x160 5620B",
|
||||
"crop": "image/jpeg 128x128 2440B",
|
||||
"optimized": "image/webp 320x200 2462B"
|
||||
},
|
||||
"thumbnail_real_jpeg_size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"original_sha256_ok": true
|
||||
},
|
||||
"job_after": {
|
||||
"job_id": "a56aa984-d3eb-4716-a47d-f246d752426c",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"error_code": "",
|
||||
"variant_rows": 3,
|
||||
"outbox_rows": 1,
|
||||
"outbox_unpublished": 0,
|
||||
"staging_removed": true
|
||||
},
|
||||
"s3": {
|
||||
"variants_objects": 3,
|
||||
"final_original": 1,
|
||||
"staging_objects": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold obsolete 5s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 1s
|
||||
timeout client 75s
|
||||
timeout server 75s
|
||||
timeout http-request 5s
|
||||
timeout http-keep-alive 10s
|
||||
timeout check 500ms
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option redispatch 1
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 init-addr none resolvers docker resolve-prefer ipv4
|
||||
server-template api 8 api:8080
|
||||
|
||||
frontend internal_stats
|
||||
bind :8404
|
||||
monitor-uri /healthz
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 5s
|
||||
@@ -0,0 +1,8 @@
|
||||
bind 0.0.0.0
|
||||
port 6379
|
||||
protected-mode yes
|
||||
save ""
|
||||
appendonly no
|
||||
maxmemory 64mb
|
||||
maxmemory-policy volatile-ttl
|
||||
include /run/secrets/valkey_auth
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, value):
|
||||
path = SECRETS / name
|
||||
if path.exists():
|
||||
print('keep secrets/' + name)
|
||||
return path.read_text().strip()
|
||||
path.write_bytes((value + '\n').encode('utf-8'))
|
||||
print('created secrets/' + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
pg = write('pg_password', secrets.token_urlsafe(24))
|
||||
write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg)
|
||||
vk_conf = SECRETS / 'valkey_auth.conf'
|
||||
if vk_conf.exists():
|
||||
vk = vk_conf.read_text().split()[-1]
|
||||
print('keep secrets/valkey_auth.conf')
|
||||
else:
|
||||
vk = secrets.token_urlsafe(24)
|
||||
write('valkey_auth.conf', 'requirepass ' + vk)
|
||||
write('cache_url', 'redis://:%s@valkey:6379/0' % vk)
|
||||
admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8))
|
||||
admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30))
|
||||
api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8))
|
||||
api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30))
|
||||
bucket = 'messenger-attachments'
|
||||
identities = {'identities': [
|
||||
{'name': 'bootstrap-admin',
|
||||
'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}],
|
||||
'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']},
|
||||
{'name': 'messenger-api',
|
||||
'credentials': [{'accessKey': api_id, 'secretKey': api_secret}],
|
||||
'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]},
|
||||
]}
|
||||
write('s3_identities.json', json.dumps(identities, indent=2))
|
||||
mq_conf = SECRETS / 'rabbitmq_auth.conf'
|
||||
if mq_conf.exists():
|
||||
mq = [l.split('=', 1)[1].strip() for l in mq_conf.read_text().splitlines() if l.startswith('default_pass')][0]
|
||||
print('keep secrets/rabbitmq_auth.conf')
|
||||
else:
|
||||
mq = secrets.token_urlsafe(24)
|
||||
write('rabbitmq_auth.conf', 'default_user = messenger\ndefault_pass = %s\ndefault_vhost = /' % mq)
|
||||
write('broker_url', 'amqp://messenger:%s@rabbitmq:5672/' % mq)
|
||||
env = ROOT / '.env'
|
||||
if not env.exists():
|
||||
shutil.copy(ROOT / '.env.example', env)
|
||||
print('created .env from .env.example')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from client import Client, Failure, check
|
||||
from image_probe import jpeg_size
|
||||
import imagegen
|
||||
from lablib import sh
|
||||
|
||||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
NETWORK = 'mtusi-lab5_data'
|
||||
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
|
||||
|
||||
|
||||
def s3tool(*args):
|
||||
cmd = ['docker', 'run', '--rm', '--network', NETWORK,
|
||||
'-v', str(ROOT / 'secrets').replace('\\', '/') + ':/secrets:ro',
|
||||
'-v', str(ROOT / 'scripts').replace('\\', '/') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8')
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError('s3tool failed: ' + (r.stdout + r.stderr).strip()[-300:])
|
||||
return json.loads(r.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-At', '-F', '|', '-v', 'ON_ERROR_STOP=1', '-c', sql)
|
||||
|
||||
|
||||
def queues():
|
||||
out = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues',
|
||||
'name', 'messages_ready', 'messages_unacknowledged', 'consumers', '--formatter', 'json')
|
||||
return {q['name']: {'ready': q['messages_ready'], 'unacked': q['messages_unacknowledged'],
|
||||
'consumers': q['consumers']} for q in json.loads(out)}
|
||||
|
||||
|
||||
def restarts(service):
|
||||
out = sh('docker', 'compose', 'ps', '-a', '-q', service)
|
||||
return {cid[:12]: int(sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid)) for cid in out.split()}
|
||||
|
||||
|
||||
def events(att_id):
|
||||
rows = psql("SELECT e.attempt, e.event, COALESCE(e.error_code,''), "
|
||||
"to_char(e.created_at AT TIME ZONE 'UTC','HH24:MI:SS.MS') "
|
||||
"FROM job_events e JOIN attachment_jobs j USING (job_id) "
|
||||
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
|
||||
return [dict(zip(('attempt', 'event', 'error_code', 'at'), r.split('|'))) for r in rows.splitlines()]
|
||||
|
||||
|
||||
def event_times(att_id):
|
||||
rows = psql("SELECT e.event, extract(epoch FROM e.created_at) FROM job_events e JOIN attachment_jobs j USING (job_id) "
|
||||
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
|
||||
return [(r.split('|')[0], float(r.split('|')[1])) for r in rows.splitlines()]
|
||||
|
||||
|
||||
def job_row(att_id):
|
||||
r = psql("SELECT j.job_id, j.status, j.attempt, j.max_attempts, COALESCE(j.error_code,''), "
|
||||
"(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id), "
|
||||
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id), "
|
||||
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL), "
|
||||
"a.staging_key IS NULL "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.attachment_id = '%s'" % att_id)
|
||||
k = ('job_id', 'status', 'attempt', 'max_attempts', 'error_code', 'variant_rows', 'outbox_rows',
|
||||
'outbox_unpublished', 'staging_removed')
|
||||
v = r.split('|')
|
||||
d = dict(zip(k, v))
|
||||
for f in ('attempt', 'max_attempts', 'variant_rows', 'outbox_rows', 'outbox_unpublished'):
|
||||
d[f] = int(d[f])
|
||||
d['staging_removed'] = d['staging_removed'] == 't'
|
||||
return d
|
||||
|
||||
|
||||
def s3_counts(chat_id, att_id):
|
||||
return {'variants_objects': s3tool('count', 'variants/%s/%s/' % (chat_id, att_id))['count'],
|
||||
'final_original': s3tool('count', 'attachments/%s/%s' % (chat_id, att_id))['count'],
|
||||
'staging_objects': s3tool('count', 'staging/' + att_id)['count']}
|
||||
|
||||
|
||||
def upload(c, actor, chat, data=PNG, name='sample.png', ctype='image/png', expected=202):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
|
||||
actor=actor, raw=data, headers={'Content-Type': ctype},
|
||||
schema='Attachment' if expected == 202 else None)
|
||||
|
||||
|
||||
def wait_status(c, actor, att_id, want, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
seen = []
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
a = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
|
||||
except Failure:
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
if not seen or seen[-1] != a['status']:
|
||||
seen.append(a['status'])
|
||||
if a['status'] in want:
|
||||
return a, seen
|
||||
time.sleep(0.3)
|
||||
raise Failure('attachment did not reach %s in %ss (seen %s)' % (want, timeout, seen))
|
||||
|
||||
|
||||
def wait_api(c, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(0.5)
|
||||
raise Failure('API not ready')
|
||||
|
||||
|
||||
def check_ready(c, actor, att):
|
||||
names = sorted(v['name'] for v in att['variants'])
|
||||
check(names == ['crop', 'optimized', 'thumbnail'], 'variants: %s' % names)
|
||||
orig = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)
|
||||
check(hashlib.sha256(orig['bytes']).hexdigest() == att['sha256'], 'original changed')
|
||||
out = {'variants': {}}
|
||||
for v in att['variants']:
|
||||
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=actor)
|
||||
check(hashlib.sha256(r['bytes']).hexdigest() == v['sha256'], v['name'] + ' checksum')
|
||||
check(r['headers'].get_content_type() == v['content_type'], v['name'] + ' content type')
|
||||
out['variants'][v['name']] = '%s %dx%d %dB' % (v['content_type'], v['width'], v['height'], v['size_bytes'])
|
||||
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=actor)['bytes']
|
||||
out['thumbnail_real_jpeg_size'] = list(jpeg_size(thumb))
|
||||
out['original_sha256_ok'] = True
|
||||
return out
|
||||
|
||||
|
||||
def fault_upload(c, actor, chat, point):
|
||||
name = 'fault-%s-%d.png' % (point, int(time.time() * 1000))
|
||||
try:
|
||||
att = upload(c, actor, chat, name=name)['json']
|
||||
return att['id'], name, 202
|
||||
except Failure as exc:
|
||||
att_id = psql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name))
|
||||
return att_id, name, 'client saw failure: ' + str(exc)[-60:]
|
||||
|
||||
|
||||
def cmd_worker_down(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
steps = []
|
||||
sh('docker', 'compose', 'stop', 'worker')
|
||||
steps.append('docker compose stop worker')
|
||||
t0 = time.monotonic()
|
||||
r = upload(c, alice, chat)
|
||||
att = r['json']
|
||||
upload_ms = round((time.monotonic() - t0) * 1000)
|
||||
steps.append('upload -> %d %s' % (r['status'], att['status']))
|
||||
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
|
||||
steps.append('message with queued attachment -> 201')
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
||||
steps.append('content while queued -> 409')
|
||||
c.message(bob, chat, 'текст работает без worker')
|
||||
steps.append('plain text message -> 201')
|
||||
time.sleep(2)
|
||||
q_before = queues()
|
||||
job_before = job_row(att['id'])
|
||||
sh('docker', 'compose', 'restart', 'api')
|
||||
sh('docker', 'compose', 'restart', 'rabbitmq')
|
||||
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 'rabbitmq', 'api', 'lb')
|
||||
wait_api(c)
|
||||
steps.append('docker compose restart api rabbitmq (volumes kept)')
|
||||
q_after = queues()
|
||||
still = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status']
|
||||
sh('docker', 'compose', 'start', 'worker')
|
||||
steps.append('docker compose start worker')
|
||||
t1 = time.monotonic()
|
||||
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
|
||||
check(ready['status'] == 'ready', 'did not become ready')
|
||||
res = {
|
||||
'attachment_id': att['id'], 'message_id': msg['id'],
|
||||
'upload_status': r['status'], 'upload_ms_worker_down': upload_ms,
|
||||
'queue_before_restart': q_before.get('attachment.process.v1'),
|
||||
'job_before_restart': job_before,
|
||||
'queue_after_api_and_broker_restart': q_after.get('attachment.process.v1'),
|
||||
'status_after_restart_before_worker': still,
|
||||
'seconds_to_ready_after_worker_start': round(time.monotonic() - t1, 2),
|
||||
'statuses_seen': seen, 'steps': steps,
|
||||
'ready_check': check_ready(c, bob, ready),
|
||||
'job_after': job_row(att['id']), 's3': s3_counts(chat['id'], att['id']),
|
||||
}
|
||||
return res
|
||||
|
||||
|
||||
def cmd_api_crash(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
before = restarts('api')
|
||||
att_id, name, client_view = fault_upload(c, alice, chat, 'api-after-commit')
|
||||
check(att_id, 'attachment row not found')
|
||||
row = job_row(att_id)
|
||||
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
|
||||
check(ready['status'] == 'ready', 'not ready')
|
||||
time.sleep(3)
|
||||
return {'fault': 'API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response',
|
||||
'attachment_id': att_id, 'client_view': client_view,
|
||||
'job_right_after_crash': row, 'api_restart_counts_before': before, 'api_restart_counts_after': restarts('api'),
|
||||
'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id),
|
||||
'ready_check': check_ready(c, bob, ready)}
|
||||
|
||||
|
||||
def _worker_crash(a, point, desc):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
before = restarts('worker')
|
||||
att_id, name, client_view = fault_upload(c, alice, chat, point)
|
||||
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
|
||||
check(ready['status'] == 'ready', 'not ready')
|
||||
time.sleep(5)
|
||||
job = job_row(att_id)
|
||||
s3 = s3_counts(chat['id'], att_id)
|
||||
check(job['variant_rows'] == 3 and s3['variants_objects'] == 3, 'duplicated or missing variants')
|
||||
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '3m')
|
||||
noop = [json.loads(l) for l in logs.splitlines() if att_id in l and 'no-op' in l]
|
||||
return {'fault': desc, 'attachment_id': att_id, 'upload': client_view,
|
||||
'worker_restart_counts_before': before, 'worker_restart_counts_after': restarts('worker'),
|
||||
'statuses_seen': seen, 'events': events(att_id), 'job_after': job, 's3': s3,
|
||||
'redelivery_noop_log': [{'result': n.get('result'), 'redelivered': n.get('redelivered')} for n in noop],
|
||||
'ready_check': check_ready(c, bob, ready)}
|
||||
|
||||
|
||||
def cmd_effect_crash(a):
|
||||
return _worker_crash(a, 'worker-after-effect',
|
||||
'worker os.Exit after writing original+variants to S3, before result COMMIT and ack')
|
||||
|
||||
|
||||
def cmd_ack_crash(a):
|
||||
return _worker_crash(a, 'worker-before-ack', 'worker os.Exit after result COMMIT + staging cleanup, before ack')
|
||||
|
||||
|
||||
def cmd_retry(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att_id, _, _ = fault_upload(c, alice, chat, 'transient')
|
||||
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
|
||||
check(ready['status'] == 'ready', 'not ready')
|
||||
t = event_times(att_id)
|
||||
fails = [x for x in t if x[0] == 'retry_scheduled']
|
||||
starts = [x for x in t if x[0] == 'started']
|
||||
gaps = [round(starts[i + 1][1] - fails[i][1], 2) for i in range(len(fails))]
|
||||
return {'fault': 'attempts 1 and 2 raise a transient error (injected_transient)',
|
||||
'attachment_id': att_id, 'statuses_seen': seen, 'events': events(att_id),
|
||||
'retry_gaps_seconds': gaps,
|
||||
'expected_gap_ranges': ['attempt1: backoff [0.5, 1.0] s + relay poll <= 1 s',
|
||||
'attempt2: backoff [1.0, 2.0] s + relay poll <= 1 s'],
|
||||
'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)}
|
||||
|
||||
|
||||
def cmd_replay(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att_id, _, _ = fault_upload(c, alice, chat, 'flaky')
|
||||
failed, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=180)
|
||||
check(failed['status'] == 'failed', 'expected failed')
|
||||
res = {'fault': 'transient error on every attempt <= JOB_MAX_ATTEMPTS', 'attachment_id': att_id,
|
||||
'statuses_seen': seen, 'failed_metadata': {k: failed[k] for k in ('status', 'error_code', 'variants')},
|
||||
'content_when_failed': c.call('GET', '/api/v1/attachments/' + att_id + '/content', 409, actor=bob)['status'],
|
||||
'attach_failed_to_message': c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
||||
body={'text': 'x', 'attachment_ids': [att_id]},
|
||||
headers={'Idempotency-Key': 'replay-' + att_id})['status'],
|
||||
'job_failed': job_row(att_id)}
|
||||
res['dlq_registry_query'] = psql("SELECT count(*) FROM attachment_jobs WHERE status = 'failed' AND attachment_id = '%s'" % att_id)
|
||||
replay = lambda: json.loads(sh('docker', 'compose', 'exec', '-T', 'worker', '/app/server', 'replay', att_id,
|
||||
'--actor', 'lab5ops'))
|
||||
res['replay_1'] = replay()
|
||||
ready, seen2 = wait_status(c, bob, att_id, ('ready', 'failed'))
|
||||
check(ready['status'] == 'ready', 'replay did not finish')
|
||||
res['replay_2_after_ready'] = replay()
|
||||
res['statuses_after_replay'] = seen2
|
||||
res['events'] = events(att_id)
|
||||
res['job_after'] = job_row(att_id)
|
||||
res['ready_check'] = check_ready(c, bob, ready)
|
||||
return res
|
||||
|
||||
|
||||
def corrupt_png():
|
||||
good = imagegen.png(64, 64)
|
||||
return (b'\x89PNG\r\n\x1a\n' + imagegen.chunk(b'IHDR', good[16:29]) +
|
||||
imagegen.chunk(b'IDAT', b'not-a-zlib-stream' * 10) + imagegen.chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def cmd_invalid(a):
|
||||
c = Client(a.base_url, timeout=60)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat, data=corrupt_png(), name='corrupt.png')['json']
|
||||
failed, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image', 'expected invalid_image')
|
||||
return {
|
||||
'corrupt_png': {'upload': 202, 'statuses_seen': seen, 'error_code': failed['error_code'],
|
||||
'events': events(att['id']), 'job': job_row(att['id'])},
|
||||
'api_rejects_before_queue': {
|
||||
'pixel_bomb_20000x20000': upload(c, alice, chat, data=imagegen.bomb_png(), name='bomb.png', expected=413)['status'],
|
||||
'not_an_image': upload(c, alice, chat, data=b'GIF89a' + b'\0' * 100, name='x.png', expected=415)['status'],
|
||||
'over_10MiB': upload(c, alice, chat, data=b'\x89PNG\r\n\x1a\n' + b'\0' * (10 * 1024 * 1024 + 1),
|
||||
name='big.png', expected=413)['status'],
|
||||
'path_in_filename': upload(c, alice, chat, name='..%2Fetc.png', expected=400)['status'],
|
||||
}}
|
||||
|
||||
|
||||
def cmd_hang(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att_id, _, _ = fault_upload(c, alice, chat, 'hang')
|
||||
t0 = time.monotonic()
|
||||
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=150)
|
||||
check(ready['status'] == 'ready', 'not ready')
|
||||
took = round(time.monotonic() - t0, 1)
|
||||
time.sleep(70)
|
||||
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '5m')
|
||||
stale = [l for l in logs.splitlines() if att_id in l and ('stale attempt' in l or 'no-op' in l)]
|
||||
return {'fault': 'attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)',
|
||||
'attachment_id': att_id, 'seconds_to_ready': took, 'statuses_seen': seen,
|
||||
'events': events(att_id), 'job_after': job_row(att_id), 's3': s3_counts(chat['id'], att_id),
|
||||
'stale_attempt_logs': [json.loads(l).get('msg') for l in stale],
|
||||
'ready_check': check_ready(c, bob, ready)}
|
||||
|
||||
|
||||
def cmd_dlq(a):
|
||||
before = queues()
|
||||
script = ('U=$(sed -n "s/^default_user = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
|
||||
'P=$(sed -n "s/^default_pass = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
|
||||
'wget -q -O- --header "Content-Type: application/json" --post-data '
|
||||
'\'{"properties":{"delivery_mode":2},"routing_key":"attachment.process.v1",'
|
||||
'"payload":"{\\"not\\":\\"a job\\"}","payload_encoding":"string"}\' '
|
||||
'"http://$U:$P@127.0.0.1:15672/api/exchanges/%2F/attachments/publish"')
|
||||
routed = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'sh', '-c', script)
|
||||
time.sleep(3)
|
||||
after = queues()
|
||||
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '1m')
|
||||
return {'published': json.loads(routed), 'dlq_before': before.get('attachment.process.dlq'),
|
||||
'dlq_after': after.get('attachment.process.dlq'),
|
||||
'worker_log': [json.loads(l)['msg'] for l in logs.splitlines() if 'DLQ' in l][-1:]}
|
||||
|
||||
|
||||
def gradient_png(w, h):
|
||||
row = b'\x02' + bytes(3 * w)
|
||||
first = b'\x00' + bytes((x * 255 // max(1, w - 1)) for x in range(w) for _ in range(3))
|
||||
raw = first + row * (h - 1)
|
||||
return imagegen.png(w, h, raw=raw, level=9)
|
||||
|
||||
|
||||
class MemSampler:
|
||||
def __init__(self, service):
|
||||
self.cids = sh('docker', 'compose', 'ps', '-q', service).split()
|
||||
self.peak = 0.0
|
||||
self._stop = threading.Event()
|
||||
self.t = threading.Thread(target=self.run, daemon=True)
|
||||
|
||||
def run(self):
|
||||
while not self._stop.is_set():
|
||||
out = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *self.cids, check=False)
|
||||
for line in out.splitlines():
|
||||
used = line.split('/')[0].strip()
|
||||
num = float(''.join(ch for ch in used if ch.isdigit() or ch == '.') or 0)
|
||||
mib = num * 1024 if used.endswith('GiB') else num / 1024 if used.endswith('KiB') else num
|
||||
self.peak = max(self.peak, mib)
|
||||
|
||||
def __enter__(self):
|
||||
self.t.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self._stop.set()
|
||||
self.t.join(timeout=10)
|
||||
|
||||
|
||||
def cmd_limits(a):
|
||||
c = Client(a.base_url, timeout=60)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
data = gradient_png(6000, 6000)
|
||||
limit = sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 'worker').split()[0])
|
||||
with MemSampler('worker') as m:
|
||||
t0 = time.monotonic()
|
||||
att = upload(c, alice, chat, data=data, name='big-6000x6000.png')['json']
|
||||
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120)
|
||||
took = round(time.monotonic() - t0, 1)
|
||||
time.sleep(2)
|
||||
check(ready['status'] == 'ready', 'big image failed: %s' % ready['error_code'])
|
||||
return {'image': '6000x6000 PNG (36 MPx, %d bytes)' % len(data), 'seconds_to_ready': took,
|
||||
'worker_peak_mem_mib': round(m.peak, 1), 'worker_mem_limit_mib': int(limit) // (1 << 20),
|
||||
'variants': {v['name']: '%dx%d' % (v['width'], v['height']) for v in ready['variants']},
|
||||
'statuses_seen': seen, 'job': job_row(att['id'])}
|
||||
|
||||
|
||||
COMMANDS = {
|
||||
'worker-down': cmd_worker_down, 'api-crash': cmd_api_crash, 'effect-crash': cmd_effect_crash,
|
||||
'ack-crash': cmd_ack_crash, 'retry': cmd_retry, 'replay': cmd_replay, 'invalid': cmd_invalid,
|
||||
'hang': cmd_hang, 'dlq': cmd_dlq, 'limits': cmd_limits,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('command', choices=sorted(COMMANDS))
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--output')
|
||||
a = p.parse_args()
|
||||
started = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
try:
|
||||
res = COMMANDS[a.command](a)
|
||||
res = {'scenario': a.command, 'result': 'PASS', 'started_at': started, **res}
|
||||
except (Failure, RuntimeError) as exc:
|
||||
res = {'scenario': a.command, 'result': 'FAIL', 'started_at': started, 'error': str(exc)}
|
||||
text = json.dumps(res, ensure_ascii=False, indent=2)
|
||||
print(text)
|
||||
if a.output:
|
||||
Path(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
|
||||
if res['result'] != 'PASS':
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
PROJECT = 'mtusi-lab5'
|
||||
|
||||
|
||||
def sh(*args, check=True):
|
||||
r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
|
||||
if check and r.returncode != 0:
|
||||
raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300]))
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def api_containers(all_states=False):
|
||||
out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc',
|
||||
'--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api',
|
||||
'--format', '{{json .}}')
|
||||
rows = []
|
||||
for line in out.splitlines():
|
||||
d = json.loads(line)
|
||||
rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']})
|
||||
return sorted(rows, key=lambda r: r['name'])
|
||||
|
||||
|
||||
def health(cid):
|
||||
return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid)
|
||||
|
||||
|
||||
def wait_healthy(cid, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if health(cid) == 'healthy':
|
||||
return True
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
|
||||
def ps_snapshot():
|
||||
return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}')
|
||||
|
||||
|
||||
class WriteLoad:
|
||||
|
||||
def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0):
|
||||
self.base = base_url
|
||||
self.workers = workers
|
||||
self.timeout = timeout
|
||||
self.pause = pause
|
||||
self.events = []
|
||||
self.confirmed = []
|
||||
self.retries = 0
|
||||
self._lock = threading.Lock()
|
||||
self._stop = threading.Event()
|
||||
self._threads = []
|
||||
c = Client(base_url, timeout=10)
|
||||
self.alice, self.bob, _eve, self.chat = c.bootstrap()
|
||||
self.reader = c
|
||||
|
||||
def _worker(self, wid):
|
||||
c = Client(self.base, timeout=self.timeout)
|
||||
key = text = None
|
||||
n = 0
|
||||
while not self._stop.is_set():
|
||||
if key is None:
|
||||
key = uuid.uuid4().hex
|
||||
text = 'load-%d-%d-%s' % (wid, n, key[:8])
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = c.message(self.alice, self.chat, text, key=key)
|
||||
with self._lock:
|
||||
self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start))
|
||||
self.confirmed.append(r['json'])
|
||||
key = None
|
||||
n += 1
|
||||
except Failure as exc:
|
||||
with self._lock:
|
||||
self.events.append((start, False, str(exc)[-60:], time.monotonic() - start))
|
||||
self.retries += 1
|
||||
time.sleep(0.05)
|
||||
if self.pause:
|
||||
time.sleep(self.pause)
|
||||
|
||||
def start(self):
|
||||
for i in range(self.workers):
|
||||
t = threading.Thread(target=self._worker, args=(i,), daemon=True)
|
||||
t.start()
|
||||
self._threads.append(t)
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
for t in self._threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
def window(self, a, b):
|
||||
ev = [e for e in self.events if a <= e[0] < b]
|
||||
ok = [e for e in ev if e[1]]
|
||||
lat = sorted(e[3] for e in ok)
|
||||
pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None
|
||||
return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok),
|
||||
'ok_rps': round(len(ok) / (b - a), 1) if b > a else None,
|
||||
'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)}
|
||||
|
||||
def verify_history(self):
|
||||
items, cursor = [], None
|
||||
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100'
|
||||
while True:
|
||||
page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''),
|
||||
actor=self.bob, schema='MessagePage')['json']
|
||||
items += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m['id']: m for m in items}
|
||||
texts = [m['text'] for m in items]
|
||||
lost = [m for m in self.confirmed if by_id.get(m['id']) != m]
|
||||
return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed),
|
||||
'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id),
|
||||
'duplicate_texts': len(texts) - len(set(texts)),
|
||||
'unique_confirmed': len({m['id'] for m in self.confirmed})}
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop lb api worker || exit /b 1
|
||||
docker compose restart postgres valkey s3 rabbitmq || exit /b 1
|
||||
docker compose up -d --no-recreate --wait postgres valkey s3 rabbitmq || exit /b 1
|
||||
docker compose start api worker lb || exit /b 1
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop lb api worker
|
||||
docker compose restart postgres valkey s3 rabbitmq
|
||||
docker compose up -d --no-recreate --wait postgres valkey s3 rabbitmq
|
||||
docker compose start api worker lb
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from urllib.parse import quote
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
HOST = os.environ.get('S3_HOST', 's3:8333')
|
||||
REGION = os.environ.get('S3_REGION', 'us-east-1')
|
||||
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
|
||||
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
|
||||
|
||||
|
||||
def creds(identity):
|
||||
if identity == 'anonymous':
|
||||
return None
|
||||
prefix = 's3_admin_' if identity == 'admin' else 's3_'
|
||||
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
|
||||
return read('access_key_id'), read('secret_access_key')
|
||||
|
||||
|
||||
def _h(key, msg):
|
||||
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||
|
||||
|
||||
_local = threading.local()
|
||||
|
||||
|
||||
def request(method, path, identity='api', body=b'', query=None):
|
||||
query = query or {}
|
||||
headers = {'host': HOST}
|
||||
payload_hash = hashlib.sha256(body).hexdigest()
|
||||
c = creds(identity)
|
||||
if c:
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
|
||||
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
|
||||
signed = ';'.join(sorted(headers))
|
||||
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
||||
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
|
||||
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
|
||||
scope = '%s/%s/s3/aws4_request' % (date, REGION)
|
||||
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
|
||||
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
|
||||
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
|
||||
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
|
||||
c[0], scope, signed, sig)
|
||||
url = quote(path, safe='/-_.~')
|
||||
if query:
|
||||
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
||||
conn = getattr(_local, 'conn', None)
|
||||
if conn is None:
|
||||
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
|
||||
try:
|
||||
conn.request(method, url, body=body or None, headers=headers)
|
||||
r = conn.getresponse()
|
||||
return r.status, r.read()
|
||||
except (OSError, http.client.HTTPException):
|
||||
_local.conn = None
|
||||
raise
|
||||
|
||||
|
||||
def obj(key):
|
||||
return '/' + BUCKET + '/' + key
|
||||
|
||||
|
||||
def list_keys(prefix, identity='api'):
|
||||
keys, token = [], None
|
||||
while True:
|
||||
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
|
||||
if token:
|
||||
q['continuation-token'] = token
|
||||
st, body = request('GET', '/' + BUCKET, identity, query=q)
|
||||
if st != 200:
|
||||
raise SystemExit('list failed: HTTP %d' % st)
|
||||
root = ET.fromstring(body)
|
||||
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
|
||||
if root.findtext(NS + 'IsTruncated') != 'true':
|
||||
return keys
|
||||
token = root.findtext(NS + 'NextContinuationToken')
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
|
||||
p.add_argument('arg', nargs='?', default='')
|
||||
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
|
||||
p.add_argument('--size', type=int, default=100)
|
||||
p.add_argument('--workers', type=int, default=32)
|
||||
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
|
||||
a = p.parse_args()
|
||||
out = {'cmd': a.cmd, 'as': a.identity}
|
||||
if a.cmd == 'get':
|
||||
st, body = request('GET', obj(a.arg), a.identity)
|
||||
out.update(status=st, bytes=len(body) if st == 200 else 0,
|
||||
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
|
||||
elif a.cmd == 'put':
|
||||
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
|
||||
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'delete':
|
||||
st, _ = request('DELETE', obj(a.arg), a.identity)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'create-bucket':
|
||||
st, _ = request('PUT', '/' + a.arg, a.identity)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'list':
|
||||
out.update(keys=list_keys(a.arg, a.identity))
|
||||
elif a.cmd == 'count':
|
||||
out.update(count=len(list_keys(a.arg, a.identity)))
|
||||
elif a.cmd in ('bulk-put', 'bulk-delete'):
|
||||
keys = [k.strip() for k in sys.stdin if k.strip()]
|
||||
data = b'x' * a.size
|
||||
|
||||
def one(k):
|
||||
if a.cmd == 'bulk-put':
|
||||
return request('PUT', obj(k), a.identity, body=data)[0]
|
||||
return request('DELETE', obj(k), a.identity)[0]
|
||||
|
||||
start = dt.datetime.now()
|
||||
with ThreadPoolExecutor(a.workers) as pool:
|
||||
codes = list(pool.map(one, keys))
|
||||
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
|
||||
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
|
||||
print(json.dumps(out))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose start worker
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose start worker
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
|
||||
set CID=
|
||||
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api"') do set CID=%%i
|
||||
if "%CID%"=="" exit /b 3
|
||||
docker stop -t 15 %CID% >nul || exit /b 1
|
||||
echo stopped api container %CID%
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=mtusi-lab5" \
|
||||
--filter "label=com.docker.compose.service=api")
|
||||
[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; }
|
||||
[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; }
|
||||
docker stop -t 15 "$cid" >/dev/null
|
||||
echo "stopped api container $cid"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop worker
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop worker
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
from client import Client, Failure
|
||||
from lab5ops import MemSampler, gradient_png, psql, queues
|
||||
import imagegen
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
e = dict(os.environ)
|
||||
e.update(env or {})
|
||||
r = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, env=e,
|
||||
capture_output=True, text=True, encoding='utf-8')
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError('compose %s: %s' % (args[:2], r.stderr[-300:]))
|
||||
|
||||
|
||||
def noisy_png(w, h):
|
||||
row_noise = lambda: b'\x00' + os.urandom(3 * w)
|
||||
row_same = b'\x02' + bytes(3 * w)
|
||||
raw = b''.join(row_noise() if y % 8 == 0 else row_same for y in range(h))
|
||||
return imagegen.png(w, h, raw=raw, level=6)
|
||||
|
||||
|
||||
SIZES = {
|
||||
'S_320x200': lambda: (ROOT / 'tests/fixtures/sample.png').read_bytes(),
|
||||
'M_2000x1500': lambda: noisy_png(2000, 1500),
|
||||
'L_5000x4000': lambda: gradient_png(5000, 4000),
|
||||
}
|
||||
COUNTS = {'S_320x200': 120, 'M_2000x1500': 40, 'L_5000x4000': 16}
|
||||
CONCURRENCY = [1, 2, 4]
|
||||
|
||||
|
||||
def pct(values, p):
|
||||
if not values:
|
||||
return None
|
||||
s = sorted(values)
|
||||
return round(s[min(len(s) - 1, int(p * len(s)))], 3)
|
||||
|
||||
|
||||
class LagSampler(threading.Thread):
|
||||
def __init__(self, chat_id):
|
||||
super().__init__(daemon=True)
|
||||
self.chat_id = chat_id
|
||||
self.samples = []
|
||||
self.stop = threading.Event()
|
||||
|
||||
def run(self):
|
||||
t0 = time.monotonic()
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
r = psql("SELECT count(*) FILTER (WHERE j.status IN ('queued','processing')), "
|
||||
"COALESCE(extract(epoch FROM now() - min(j.created_at) FILTER (WHERE j.status IN ('queued','processing'))), 0) "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % self.chat_id)
|
||||
active, oldest = r.split('|')
|
||||
q = queues().get('attachment.process.v1', {})
|
||||
self.samples.append({'t': round(time.monotonic() - t0, 1), 'active': int(active),
|
||||
'mq_ready': q.get('ready'), 'mq_unacked': q.get('unacked'),
|
||||
'oldest_age_s': round(float(oldest), 2)})
|
||||
except Exception:
|
||||
pass
|
||||
self.stop.wait(0.5)
|
||||
|
||||
|
||||
def burst(c, actor, chat, data, n, threads=8):
|
||||
codes = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def one(i):
|
||||
try:
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=t1-%d.png' % i, [202, 503],
|
||||
actor=actor, raw=data, headers={'Content-Type': 'image/png'})
|
||||
code = r['status']
|
||||
ra = r['headers'].get('Retry-After')
|
||||
except Failure as exc:
|
||||
code, ra = 'error:' + str(exc)[-40:], None
|
||||
with lock:
|
||||
codes.append((code, ra))
|
||||
|
||||
t0 = time.monotonic()
|
||||
with ThreadPoolExecutor(threads) as pool:
|
||||
list(pool.map(one, range(n)))
|
||||
return codes, time.monotonic() - t0
|
||||
|
||||
|
||||
def wait_drained(chat_id, timeout=900):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
n = int(psql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
||||
"WHERE a.chat_id = '%s' AND j.status IN ('queued','processing')" % chat_id))
|
||||
if n == 0:
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def job_stats(chat_id):
|
||||
rows = psql("SELECT j.status, extract(epoch FROM j.created_at), extract(epoch FROM j.finished_at), j.attempt "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % chat_id)
|
||||
jobs = [r.split('|') for r in rows.splitlines()]
|
||||
ready = [(float(c), float(f)) for s, c, f, _ in jobs if s == 'ready']
|
||||
ttr = [f - c for c, f in ready]
|
||||
span = (max(f for _, f in ready) - min(c for c, _ in ready)) if ready else 0
|
||||
return {'jobs': len(jobs), 'ready': len(ready), 'failed': sum(1 for j in jobs if j[0] == 'failed'),
|
||||
'retried_jobs': sum(1 for j in jobs if int(j[3]) > 1),
|
||||
'throughput_jobs_per_s': round(len(ready) / span, 2) if span else None,
|
||||
'pipeline_span_s': round(span, 2),
|
||||
'time_to_ready_s': {'p50': pct(ttr, .5), 'p95': pct(ttr, .95), 'max': round(max(ttr), 3) if ttr else None}}
|
||||
|
||||
|
||||
def set_worker(concurrency):
|
||||
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_CONCURRENCY': str(concurrency)})
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def one_run(base_url, size, conc):
|
||||
c = Client(base_url, timeout=60)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
data = SIZES[size]()
|
||||
n = COUNTS[size]
|
||||
lag = LagSampler(chat['id'])
|
||||
with MemSampler('worker') as mem:
|
||||
lag.start()
|
||||
codes, upload_s = burst(c, alice, chat, data, n)
|
||||
drained = wait_drained(chat['id'])
|
||||
time.sleep(1)
|
||||
lag.stop.set()
|
||||
lag.join()
|
||||
st = job_stats(chat['id'])
|
||||
s = lag.samples
|
||||
return {'size': size, 'image_bytes': len(data), 'worker_concurrency_prefetch': conc, 'uploads': n,
|
||||
'accepted_202': sum(1 for x, _ in codes if x == 202),
|
||||
'rejected': sum(1 for x, _ in codes if x != 202),
|
||||
'upload_burst_s': round(upload_s, 2), 'drained': drained, **st,
|
||||
'queue_lag': {'max_active_jobs': max((x['active'] for x in s), default=None),
|
||||
'max_mq_ready': max((x['mq_ready'] or 0 for x in s), default=None),
|
||||
'max_oldest_job_age_s': max((x['oldest_age_s'] for x in s), default=None)},
|
||||
'worker_peak_rss_mib': round(mem.peak, 1),
|
||||
'lag_timeline': s[::max(1, len(s) // 40)]}
|
||||
|
||||
|
||||
def cmd_matrix(a):
|
||||
runs = []
|
||||
try:
|
||||
for conc in CONCURRENCY:
|
||||
set_worker(conc)
|
||||
for size in SIZES:
|
||||
r = one_run(a.base_url, size, conc)
|
||||
print(json.dumps({k: r[k] for k in ('size', 'worker_concurrency_prefetch', 'throughput_jobs_per_s',
|
||||
'time_to_ready_s', 'worker_peak_rss_mib', 'queue_lag')}), flush=True)
|
||||
runs.append(r)
|
||||
finally:
|
||||
set_worker(2)
|
||||
return {'runs': runs}
|
||||
|
||||
|
||||
def cmd_backpressure(a):
|
||||
bound = 20
|
||||
compose('up', '-d', '--no-deps', '--wait', 'api', env={'MAX_ACTIVE_JOBS': str(bound)})
|
||||
set_worker(1)
|
||||
try:
|
||||
c = Client(a.base_url, timeout=60)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
data = SIZES['M_2000x1500']()
|
||||
lag = LagSampler(chat['id'])
|
||||
lag.start()
|
||||
t0 = time.monotonic()
|
||||
codes, upload_s = burst(c, alice, chat, data, 80)
|
||||
drained = wait_drained(chat['id'])
|
||||
drain_s = time.monotonic() - t0
|
||||
t1 = time.monotonic()
|
||||
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after.png', 202, actor=alice,
|
||||
raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
|
||||
while True:
|
||||
st = c.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']['status']
|
||||
if st in ('ready', 'failed'):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
recovery_ready_s = time.monotonic() - t1
|
||||
time.sleep(1)
|
||||
lag.stop.set()
|
||||
lag.join()
|
||||
s = lag.samples
|
||||
return {'max_active_jobs_bound': bound, 'worker_concurrency': 1, 'uploads': 80,
|
||||
'accepted_202': sum(1 for x, _ in codes if x == 202),
|
||||
'rejected_503': sum(1 for x, _ in codes if x == 503),
|
||||
'retry_after_header': sorted({ra for x, ra in codes if x == 503}),
|
||||
'other': [x for x, _ in codes if x not in (202, 503)],
|
||||
'upload_burst_s': round(upload_s, 2), 'drained': drained, 'drain_s': round(drain_s, 1),
|
||||
'max_active_observed': max((x['active'] for x in s), default=None),
|
||||
'jobs': job_stats(chat['id']), 'after_overload_upload_status': st,
|
||||
'after_overload_seconds_to_ready': round(recovery_ready_s, 2),
|
||||
'lag_timeline': s[::max(1, len(s) // 40)]}
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', '--wait', 'api')
|
||||
set_worker(2)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('command', choices=['matrix', 'backpressure'])
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--output')
|
||||
a = p.parse_args()
|
||||
env = {'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'worker_limits': 'cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB',
|
||||
'host': 'Windows 11, Docker Desktop 29.7.2 (WSL2)'}
|
||||
res = {'experiment': a.command, **env, **(cmd_matrix(a) if a.command == 'matrix' else cmd_backpressure(a))}
|
||||
text = json.dumps(res, ensure_ascii=False, indent=1)
|
||||
if a.output:
|
||||
(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
|
||||
print(text[:3000])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,403 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/internal/api"
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/broker"
|
||||
"mtusi-messenger/internal/cache"
|
||||
"mtusi-messenger/internal/db"
|
||||
"mtusi-messenger/internal/imaging"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/relay"
|
||||
"mtusi-messenger/internal/store"
|
||||
"mtusi-messenger/internal/worker"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
func getenv(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func getint(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n <= 0 {
|
||||
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
|
||||
os.Exit(2)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func secret(key string) (string, error) {
|
||||
if path := os.Getenv(key + "_FILE"); path != "" {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s_FILE: %w", key, err)
|
||||
}
|
||||
return strings.TrimSpace(string(b)), nil
|
||||
}
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v, nil
|
||||
}
|
||||
return "", fmt.Errorf("%s or %s_FILE is required", key, key)
|
||||
}
|
||||
|
||||
func main() {
|
||||
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if err := run(log); err != nil {
|
||||
log.Error("fatal", "error", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func openDB(ctx context.Context) (*store.Store, func(), error) {
|
||||
dsn, err := secret("DATABASE_URL")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))})
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("database config is invalid")
|
||||
}
|
||||
return store.New(pool), pool.Close, nil
|
||||
}
|
||||
|
||||
func getduration(key string, def time.Duration) time.Duration {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
|
||||
os.Exit(2)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func openObjects() (*objstore.Store, error) { return openObjectsAs("S3") }
|
||||
|
||||
func openObjectsAs(prefix string) (*objstore.Store, error) {
|
||||
access, err := secret(prefix + "_ACCESS_KEY_ID")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secretKey, err := secret(prefix + "_SECRET_ACCESS_KEY")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return objstore.New(objstore.Config{
|
||||
Endpoint: getenv("S3_ENDPOINT", "http://s3:9000"),
|
||||
Region: getenv("S3_REGION", "us-east-1"),
|
||||
Bucket: getenv("S3_BUCKET", "messenger-attachments"),
|
||||
AccessKey: access,
|
||||
SecretKey: secretKey,
|
||||
})
|
||||
}
|
||||
|
||||
func getbool(key string) bool {
|
||||
v := strings.ToLower(os.Getenv(key))
|
||||
return v == "1" || v == "true" || v == "yes"
|
||||
}
|
||||
|
||||
func startRelay(ctx context.Context, st *store.Store, log *slog.Logger) (*relay.Relay, func(), error) {
|
||||
url, err := secret("BROKER_URL")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pub := broker.NewPublisher(url)
|
||||
rl := relay.New(st, pub, log, getduration("OUTBOX_POLL_INTERVAL", time.Second))
|
||||
rctx, cancel := context.WithCancel(ctx)
|
||||
go rl.Run(rctx)
|
||||
return rl, func() { cancel(); pub.Close() }, nil
|
||||
}
|
||||
|
||||
func runWorker(ctx context.Context, log *slog.Logger) error {
|
||||
st, closeDB, err := openDB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB()
|
||||
objects, err := openObjects()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
url, err := secret("BROKER_URL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
budget := int64(getint("WORKER_DECODE_BUDGET_BYTES", 384<<20))
|
||||
cfg := worker.Config{
|
||||
BrokerURL: url,
|
||||
Concurrency: getint("WORKER_CONCURRENCY", 2),
|
||||
MaxAttempts: getint("JOB_MAX_ATTEMPTS", 5),
|
||||
JobTimeout: time.Duration(getint("JOB_TIMEOUT_SECONDS", 60)) * time.Second,
|
||||
Lease: time.Duration(getint("JOB_LEASE_SECONDS", 30)) * time.Second,
|
||||
BackoffBase: getduration("JOB_BACKOFF_BASE", time.Second),
|
||||
BackoffMax: getduration("JOB_BACKOFF_MAX", time.Minute),
|
||||
StagingRetain: time.Duration(getint("STAGING_RETENTION_SECONDS", 86400)) * time.Second,
|
||||
PendingGrace: getduration("PENDING_UPLOAD_GRACE", 15*time.Minute),
|
||||
FaultInjection: getbool("FAULT_INJECTION"),
|
||||
SweepEvery: getduration("SWEEP_INTERVAL", 5*time.Second),
|
||||
Limits: imaging.Limits{
|
||||
MaxBytes: int64(getint("MAX_UPLOAD_BYTES", 10485760)),
|
||||
MaxPixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)),
|
||||
Budget: semaphore.NewWeighted(budget),
|
||||
BudgetSize: budget,
|
||||
},
|
||||
}
|
||||
if cfg.Lease < 6*time.Second {
|
||||
return errors.New("JOB_LEASE_SECONDS must be >= 6")
|
||||
}
|
||||
wctx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
if !getbool("WORKER_NO_RELAY") {
|
||||
_, stopRelay, err := startRelay(wctx, st, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopRelay()
|
||||
}
|
||||
log.Info("worker started", "concurrency", cfg.Concurrency, "max_attempts", cfg.MaxAttempts,
|
||||
"job_timeout_s", int(cfg.JobTimeout.Seconds()), "lease_s", int(cfg.Lease.Seconds()),
|
||||
"fault_injection", cfg.FaultInjection, "build", getenv("APP_BUILD", "dev"))
|
||||
return worker.New(cfg, st, objects, log).Run(wctx)
|
||||
}
|
||||
|
||||
func runReplay(ctx context.Context, args []string) error {
|
||||
fs := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||
actor := fs.String("actor", "operator", "who requested the replay (audit)")
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
args = append(args[1:], args[0])
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if fs.NArg() != 1 {
|
||||
return errors.New("usage: replay <attachment_id> [--actor name]")
|
||||
}
|
||||
st, closeDB, err := openDB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB()
|
||||
jobID, attempt, err := st.ReplayFailed(ctx, fs.Arg(0), *actor, getint("JOB_MAX_ATTEMPTS", 5))
|
||||
switch {
|
||||
case errors.Is(err, store.ErrJobAlreadyOK):
|
||||
fmt.Println(`{"result":"noop","reason":"job already ready; replay refused"}`)
|
||||
return nil
|
||||
case errors.Is(err, store.ErrJobNotFailed):
|
||||
fmt.Println(`{"result":"noop","reason":"job is queued/processing, not failed"}`)
|
||||
return nil
|
||||
case errors.Is(err, store.ErrStagingGone):
|
||||
fmt.Println(`{"result":"refused","reason":"staging original removed by retention"}`)
|
||||
return nil
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
out, _ := json.Marshal(map[string]any{"result": "replayed", "job_id": jobID, "attempt": attempt})
|
||||
fmt.Println(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
func run(log *slog.Logger) error {
|
||||
ctx := context.Background()
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
||||
dsn, err := secret("DATABASE_URL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2})
|
||||
if err != nil {
|
||||
return errors.New("database config is invalid")
|
||||
}
|
||||
defer pool.Close()
|
||||
mctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
return db.Migrate(mctx, pool, log)
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "s3-init" {
|
||||
objects, err := openObjectsAs("S3_ADMIN")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var created bool
|
||||
for attempt := 1; ; attempt++ {
|
||||
bctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
created, err = objects.EnsureBucket(bctx, getenv("S3_REGION", "us-east-1"))
|
||||
cancel()
|
||||
if err == nil || attempt == 30 {
|
||||
break
|
||||
}
|
||||
log.Warn("s3 not ready, retrying", "attempt", attempt, "error", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("bucket ready", "bucket", objects.Bucket(), "created", created)
|
||||
return nil
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "worker" {
|
||||
return runWorker(ctx, log)
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "replay" {
|
||||
return runReplay(ctx, os.Args[2:])
|
||||
}
|
||||
|
||||
st, closeDB, err := openDB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB()
|
||||
|
||||
ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second
|
||||
cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second
|
||||
|
||||
var sessCache auth.SessionCache
|
||||
switch mode := getenv("SESSION_STORE", "postgres"); mode {
|
||||
case "postgres":
|
||||
case "valkey", "redis":
|
||||
url, err := secret("CACHE_URL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vk, err := cache.NewValkey(url)
|
||||
if err != nil {
|
||||
return errors.New("cache config is invalid")
|
||||
}
|
||||
defer vk.Close()
|
||||
sessCache = vk
|
||||
default:
|
||||
return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode)
|
||||
}
|
||||
|
||||
hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4))
|
||||
sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log)
|
||||
|
||||
instanceID := getenv("INSTANCE_ID", "")
|
||||
if instanceID == "" {
|
||||
h, err := os.Hostname()
|
||||
if err != nil || h == "" {
|
||||
return errors.New("cannot determine instance id; set INSTANCE_ID")
|
||||
}
|
||||
instanceID = h
|
||||
}
|
||||
idemTTL := time.Duration(getint("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second
|
||||
objects, err := openObjects()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maxUpload := int64(getint("MAX_UPLOAD_BYTES", 10485760))
|
||||
if maxUpload > 10485760 {
|
||||
return errors.New("MAX_UPLOAD_BYTES above the 10 MiB contract limit")
|
||||
}
|
||||
drainDelay := time.Duration(getint("SHUTDOWN_DRAIN_MS", 3000)) * time.Millisecond
|
||||
|
||||
rl, stopRelay, err := startRelay(ctx, st, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopRelay()
|
||||
|
||||
ready := &atomic.Bool{}
|
||||
ready.Store(true)
|
||||
|
||||
host := getenv("APP_HOST", "0.0.0.0")
|
||||
port := getenv("APP_PORT", "8080")
|
||||
addr := host + ":" + port
|
||||
srv := api.NewServer(st, objects, sessions, hasher, log, ready, api.Options{
|
||||
InstanceID: instanceID, IdempotencyTTL: idemTTL,
|
||||
MaxUploadBytes: maxUpload,
|
||||
MaxImagePixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)),
|
||||
UploadConcurrency: int64(getint("UPLOAD_CONCURRENCY", 8)),
|
||||
UploadTimeout: getduration("UPLOAD_TIMEOUT", 60*time.Second),
|
||||
S3Timeout: getduration("S3_TIMEOUT", 15*time.Second),
|
||||
Relay: rl,
|
||||
MaxActiveJobs: getint("MAX_ACTIVE_JOBS", 500),
|
||||
PipelineVersion: imaging.PipelineVersion,
|
||||
MaxAttempts: getint("JOB_MAX_ATTEMPTS", 5),
|
||||
FaultInjection: getbool("FAULT_INJECTION"),
|
||||
})
|
||||
httpSrv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: srv.Handler(),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 90 * time.Second,
|
||||
WriteTimeout: 90 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
bgCtx, stopBg := context.WithCancel(ctx)
|
||||
defer stopBg()
|
||||
go func() {
|
||||
t := time.NewTicker(10 * time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-bgCtx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
c, cancel := context.WithTimeout(bgCtx, 10*time.Second)
|
||||
if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 {
|
||||
log.Info("expired sessions purged", "count", n)
|
||||
}
|
||||
if n, err := st.PurgeIdempotency(c, idemTTL); err == nil && n > 0 {
|
||||
log.Info("expired idempotency keys purged", "count", n)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- httpSrv.ListenAndServe() }()
|
||||
log.Info("server started", "addr", addr, "pid", os.Getpid(), "instance_id", instanceID, "build", getenv("APP_BUILD", "dev"), "s3_bucket", objects.Bucket(),
|
||||
"session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds()))
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
case sig := <-stop:
|
||||
log.Info("shutdown signal received", "signal", sig.String())
|
||||
ready.Store(false)
|
||||
time.Sleep(drainDelay)
|
||||
httpSrv.SetKeepAlivesEnabled(false)
|
||||
sctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := httpSrv.Shutdown(sctx); err != nil {
|
||||
log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error())
|
||||
_ = httpSrv.Close()
|
||||
}
|
||||
log.Info("server stopped cleanly")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
module mtusi-messenger
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.7.2
|
||||
github.com/minio/minio-go/v7 v7.0.82
|
||||
github.com/redis/go-redis/v9 v9.7.0
|
||||
golang.org/x/crypto v0.31.0
|
||||
golang.org/x/sync v0.10.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.8.1 // indirect
|
||||
github.com/gen2brain/webp v0.5.2 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tetratelabs/wazero v1.8.1 // indirect
|
||||
golang.org/x/image v0.23.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
)
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE=
|
||||
github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/gen2brain/webp v0.5.2 h1:aYdjbU/2L98m+bqUdkYMOIY93YC+EN3HuZLMaqgMD9U=
|
||||
github.com/gen2brain/webp v0.5.2/go.mod h1:Nb3xO5sy6MeUAHhru9H3GT7nlOQO5dKRNNlE92CZrJw=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
|
||||
github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
|
||||
github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
|
||||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
|
||||
github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
|
||||
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,334 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
mimePNG = "image/png"
|
||||
mimeJPEG = "image/jpeg"
|
||||
|
||||
maxDrainBytes = 64 << 20
|
||||
)
|
||||
|
||||
func (s *Server) objectError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
s.log.Warn("object storage unavailable", "request_id", requestID(r), "error", err.Error())
|
||||
apperr.Write(w, requestID(r), apperr.Unavailable("attachment storage temporarily unavailable"))
|
||||
}
|
||||
|
||||
func validFilename(name string) bool {
|
||||
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || name == "." || name == ".." {
|
||||
return false
|
||||
}
|
||||
if n := runeLen(name); n < 1 || n > 255 {
|
||||
return false
|
||||
}
|
||||
for _, c := range name {
|
||||
switch {
|
||||
case c == '/', c == '\\', c == 0, unicode.IsControl(c):
|
||||
return false
|
||||
case c >= 0x202a && c <= 0x202e, c >= 0x2066 && c <= 0x2069:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func sniffImage(data []byte) string {
|
||||
switch {
|
||||
case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")):
|
||||
return mimePNG
|
||||
case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}):
|
||||
return mimeJPEG
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type imageInfo struct {
|
||||
contentType string
|
||||
width, height int
|
||||
}
|
||||
|
||||
func (s *Server) inspectImage(data []byte, declared string) (imageInfo, *apperr.AppError) {
|
||||
detected := sniffImage(data)
|
||||
if detected == "" {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("content is not a PNG or JPEG image")
|
||||
}
|
||||
if detected != declared {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("Content-Type does not match the image content")
|
||||
}
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || (format != "png" && format != "jpeg") {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("image header is invalid")
|
||||
}
|
||||
pixels := int64(cfg.Width) * int64(cfg.Height)
|
||||
if pixels > s.maxPixels {
|
||||
return imageInfo{}, apperr.PayloadTooLarge("image exceeds " + strconv.FormatInt(s.maxPixels, 10) + " pixels")
|
||||
}
|
||||
return imageInfo{contentType: detected, width: cfg.Width, height: cfg.Height}, nil
|
||||
}
|
||||
|
||||
func readBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, *apperr.AppError) {
|
||||
var buf bytes.Buffer
|
||||
if r.ContentLength > 0 {
|
||||
buf.Grow(int(r.ContentLength))
|
||||
}
|
||||
_, err := buf.ReadFrom(http.MaxBytesReader(w, r.Body, limit))
|
||||
var maxErr *http.MaxBytesError
|
||||
switch {
|
||||
case errors.As(err, &maxErr):
|
||||
return nil, apperr.PayloadTooLarge("attachment exceeds " + strconv.FormatInt(limit, 10) + " bytes")
|
||||
case err != nil:
|
||||
return nil, apperr.InvalidRequest("request body could not be read")
|
||||
case buf.Len() == 0:
|
||||
return nil, apperr.InvalidRequest("request body is empty")
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
body := r.Body
|
||||
defer func() { _, _ = io.CopyN(io.Discard, body, maxDrainBytes) }()
|
||||
|
||||
id := requestID(r)
|
||||
chatID, ok := s.chatAccess(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || len(query["filename"]) != 1 {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("exactly one filename query parameter is required"))
|
||||
return
|
||||
}
|
||||
filename := query.Get("filename")
|
||||
if !validFilename(filename) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("filename must be 1..255 code points without path separators or control characters"))
|
||||
return
|
||||
}
|
||||
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if media != mimePNG && media != mimeJPEG {
|
||||
apperr.Write(w, id, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
|
||||
return
|
||||
}
|
||||
if r.ContentLength > s.maxUpload {
|
||||
apperr.Write(w, id, apperr.PayloadTooLarge("attachment exceeds "+strconv.FormatInt(s.maxUpload, 10)+" bytes"))
|
||||
return
|
||||
}
|
||||
|
||||
if s.maxActiveJobs > 0 {
|
||||
n, err := s.store.ActiveJobs(r.Context())
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if n >= s.maxActiveJobs {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
apperr.Write(w, id, apperr.Unavailable("attachment processing queue is full, retry later"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.uploadSem.Acquire(r.Context(), 1); err != nil {
|
||||
apperr.Write(w, id, apperr.Unavailable("too many concurrent uploads, retry later"))
|
||||
return
|
||||
}
|
||||
defer s.uploadSem.Release(1)
|
||||
|
||||
data, aerr := readBody(w, r, s.maxUpload)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
info, aerr := s.inspectImage(data, media)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
attID := uuid.NewString()
|
||||
att := &model.Attachment{
|
||||
ID: attID, ChatID: chatID, OwnerID: currentUser(r).ID, Filename: filename,
|
||||
ContentType: info.contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]),
|
||||
Status: "queued", Variants: []model.Variant{},
|
||||
|
||||
ObjectKey: "attachments/" + chatID + "/" + attID,
|
||||
}
|
||||
stagingKey := "staging/" + attID
|
||||
|
||||
if err := s.store.InsertPendingAttachment(r.Context(), att, info.width, info.height, stagingKey); err != nil {
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return
|
||||
}
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
putCtx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
|
||||
err = s.objects.Put(putCtx, stagingKey, data, att.ContentType)
|
||||
cancel()
|
||||
if err != nil {
|
||||
go s.compensateUpload(att, stagingKey, id)
|
||||
s.objectError(w, r, err)
|
||||
return
|
||||
}
|
||||
_, updated, err := s.store.QueueAttachment(r.Context(), att.ID, s.pipelineVersion, s.maxAttempts, s.maxActiveJobs)
|
||||
if err != nil {
|
||||
go s.compensateUpload(att, stagingKey, id)
|
||||
if errors.Is(err, store.ErrQueueFull) {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
apperr.Write(w, id, apperr.Unavailable("attachment processing queue is full, retry later"))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.Unavailable("upload expired, retry"))
|
||||
return
|
||||
}
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if s.faultInjection && strings.HasPrefix(filename, "fault-api-after-commit") {
|
||||
s.log.Warn("FAULT: exit after COMMIT before publish/response", "attachment_id", att.ID)
|
||||
os.Exit(42)
|
||||
}
|
||||
if s.relay != nil {
|
||||
s.relay.Kick()
|
||||
}
|
||||
att.UpdatedAt = model.NewTime(updated)
|
||||
writeJSON(w, r, http.StatusAccepted, att)
|
||||
}
|
||||
|
||||
func (s *Server) compensateUpload(att *model.Attachment, stagingKey, reqID string) {
|
||||
dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer dcancel()
|
||||
deleted, err := s.store.DeletePendingAttachment(dctx, att.ID)
|
||||
if err != nil || !deleted {
|
||||
if err != nil {
|
||||
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
|
||||
"attachment_id", att.ID, "stage", "delete_row")
|
||||
}
|
||||
return
|
||||
}
|
||||
octx, ocancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer ocancel()
|
||||
if err := s.objects.Remove(octx, stagingKey); err != nil {
|
||||
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
|
||||
"attachment_id", att.ID, "stage", "remove_object")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) attachmentForRequest(w http.ResponseWriter, r *http.Request) (*model.Attachment, bool) {
|
||||
id := requestID(r)
|
||||
attID, err := parseUUID(r.PathValue("attachment_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("attachment_id must be a UUID"))
|
||||
return nil, false
|
||||
}
|
||||
att, err := s.store.AttachmentForUser(r.Context(), attID, currentUser(r).ID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("attachment not found or not accessible"))
|
||||
return nil, false
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return nil, false
|
||||
}
|
||||
return att, true
|
||||
}
|
||||
|
||||
func (s *Server) getAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
if att, ok := s.attachmentForRequest(w, r); ok {
|
||||
writeJSON(w, r, http.StatusOK, att)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) downloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
id := requestID(r)
|
||||
att, ok := s.attachmentForRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
variant := r.URL.Query().Get("variant")
|
||||
switch variant {
|
||||
case "", "original", "thumbnail", "crop", "optimized":
|
||||
default:
|
||||
apperr.Write(w, id, apperr.InvalidRequest("unknown variant"))
|
||||
return
|
||||
}
|
||||
if att.Status != "ready" {
|
||||
apperr.Write(w, id, apperr.Conflict("attachment is "+att.Status+", content is not available yet"))
|
||||
return
|
||||
}
|
||||
key, contentType, etag := att.ObjectKey, att.ContentType, att.SHA256
|
||||
if variant != "" && variant != "original" {
|
||||
found := false
|
||||
for _, v := range att.Variants {
|
||||
if v.Name == variant {
|
||||
key, contentType, etag, found = v.ObjectKey, v.ContentType, v.SHA256, true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
apperr.Write(w, id, apperr.NotFound("variant not found"))
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
|
||||
defer cancel()
|
||||
obj, size, err := s.objects.Get(ctx, key)
|
||||
if objstore.IsNotFound(err) {
|
||||
s.log.Error("attachment object missing in S3", "request_id", id, "attachment_id", att.ID)
|
||||
apperr.Write(w, id, apperr.NotFound("attachment content is not available"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.objectError(w, r, err)
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
h := w.Header()
|
||||
h.Set("Content-Type", contentType)
|
||||
h.Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("Cache-Control", "private, no-store")
|
||||
h.Set("Content-Disposition", "inline; filename*=UTF-8''"+url.PathEscape(att.Filename))
|
||||
h.Set("ETag", `"`+etag+`"`)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, obj); err != nil {
|
||||
s.log.Warn("download interrupted", "request_id", id, "attachment_id", att.ID, "error", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
id := requestID(r)
|
||||
if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) {
|
||||
s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error())
|
||||
apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable"))
|
||||
return
|
||||
}
|
||||
s.log.Error("unexpected storage error", "request_id", id, "error", err.Error())
|
||||
apperr.Write(w, id, apperr.Internal("internal error"))
|
||||
}
|
||||
|
||||
func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
|
||||
}
|
||||
|
||||
func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.ready.Load() {
|
||||
apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down"))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Second)
|
||||
defer cancel()
|
||||
if err := s.store.Ping(ctx); err != nil {
|
||||
apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable"))
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
|
||||
}
|
||||
|
||||
type createUserReq struct {
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func validPassword(p string) bool {
|
||||
n := utf8.RuneCountInString(p)
|
||||
return utf8.ValidString(p) && n >= 12 && n <= 128
|
||||
}
|
||||
|
||||
func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
id := requestID(r)
|
||||
var req createUserReq
|
||||
if aerr := decodeJSON(w, r, &req); aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
if !usernameRe.MatchString(req.Username) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$"))
|
||||
return
|
||||
}
|
||||
if !validText(req.DisplayName, 100) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points"))
|
||||
return
|
||||
}
|
||||
if !validPassword(req.Password) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points"))
|
||||
return
|
||||
}
|
||||
hash, err := s.hasher.Hash(r.Context(), req.Password)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash)
|
||||
if errors.Is(err, store.ErrUsernameTaken) {
|
||||
apperr.Write(w, id, apperr.Conflict("username is already taken"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, u)
|
||||
}
|
||||
|
||||
func (s *Server) me(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, currentUser(r))
|
||||
}
|
||||
|
||||
const basicChallenge = `Basic realm="messenger", charset="UTF-8"`
|
||||
|
||||
func unauthorizedBasic(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("WWW-Authenticate", basicChallenge)
|
||||
apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password"))
|
||||
}
|
||||
|
||||
func parseBasic(r *http.Request) (string, string, bool) {
|
||||
scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Basic") {
|
||||
return "", "", false
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value))
|
||||
if err != nil || !utf8.Valid(raw) {
|
||||
return "", "", false
|
||||
}
|
||||
user, pass, ok := strings.Cut(string(raw), ":")
|
||||
if !ok || !usernameRe.MatchString(user) || !validPassword(pass) {
|
||||
return "", "", false
|
||||
}
|
||||
return user, pass, true
|
||||
}
|
||||
|
||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
username, password, ok := parseBasic(r)
|
||||
if !ok {
|
||||
unauthorizedBasic(w, r)
|
||||
return
|
||||
}
|
||||
user, hash, err := s.store.Credentials(r.Context(), username)
|
||||
if err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
valid, err := s.hasher.Verify(r.Context(), password, hash)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if !valid {
|
||||
unauthorizedBasic(w, r)
|
||||
return
|
||||
}
|
||||
token, expires, err := s.sessions.Create(r.Context(), user)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, model.Session{
|
||||
Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodDelete {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
err := s.sessions.Revoke(r.Context(), currentToken(r))
|
||||
if errors.Is(err, auth.ErrInvalidSession) {
|
||||
unauthorizedBearer(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Request-Id", requestID(r))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
type createChatReq struct {
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
}
|
||||
|
||||
func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
s.createChat(w, r)
|
||||
case http.MethodGet:
|
||||
s.listChats(w, r)
|
||||
default:
|
||||
methodNotAllowed(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) createChat(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
var req createChatReq
|
||||
if aerr := decodeJSON(w, r, &req); aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
if !validText(req.Title, 100) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points"))
|
||||
return
|
||||
}
|
||||
if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items"))
|
||||
return
|
||||
}
|
||||
seen := make(map[string]bool, len(req.MemberIDs))
|
||||
for _, raw := range req.MemberIDs {
|
||||
uid, err := parseUUID(raw)
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings"))
|
||||
return
|
||||
}
|
||||
if seen[uid] {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id"))
|
||||
return
|
||||
}
|
||||
seen[uid] = true
|
||||
}
|
||||
caller := currentUser(r)
|
||||
if !seen[caller.ID] {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids"))
|
||||
return
|
||||
}
|
||||
|
||||
chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs)
|
||||
if errors.Is(err, store.ErrUnknownMember) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("unknown member_id"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, chat)
|
||||
}
|
||||
|
||||
func (s *Server) listChats(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
caller := currentUser(r)
|
||||
limit, cur, aerr := pagingParams(r, caller.ID)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
page := model.Page[*model.Chat]{Items: items}
|
||||
if more && len(items) > 0 {
|
||||
last := items[len(items)-1]
|
||||
encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID})
|
||||
page.NextCursor = &encoded
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
id := requestID(r)
|
||||
chatID, err := parseUUID(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
|
||||
return
|
||||
}
|
||||
chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, chat)
|
||||
}
|
||||
|
||||
type createMessageReq struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
s.createMessage(w, r)
|
||||
case http.MethodGet:
|
||||
s.listMessages(w, r)
|
||||
default:
|
||||
methodNotAllowed(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
id := requestID(r)
|
||||
chatID, err := parseUUID(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
|
||||
return "", false
|
||||
}
|
||||
ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return "", false
|
||||
}
|
||||
if !ok {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return "", false
|
||||
}
|
||||
return chatID, true
|
||||
}
|
||||
|
||||
func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
chatID, err := parseUUID(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
|
||||
return
|
||||
}
|
||||
|
||||
reject := func(aerr *apperr.AppError) {
|
||||
if _, ok := s.chatAccess(w, r); !ok {
|
||||
return
|
||||
}
|
||||
apperr.Write(w, id, aerr)
|
||||
}
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if !idempotencyKeyRe.MatchString(key) {
|
||||
reject(apperr.InvalidRequest("Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}"))
|
||||
return
|
||||
}
|
||||
var req createMessageReq
|
||||
if aerr := decodeJSON(w, r, &req); aerr != nil {
|
||||
reject(aerr)
|
||||
return
|
||||
}
|
||||
if req.AttachmentIDs == nil {
|
||||
req.AttachmentIDs = []string{}
|
||||
}
|
||||
if len(req.AttachmentIDs) > 10 {
|
||||
reject(apperr.InvalidRequest("attachment_ids must contain at most 10 items"))
|
||||
return
|
||||
}
|
||||
seen := make(map[string]bool, len(req.AttachmentIDs))
|
||||
for _, raw := range req.AttachmentIDs {
|
||||
aid, err := parseUUID(raw)
|
||||
if err != nil {
|
||||
reject(apperr.InvalidRequest("attachment_ids must be UUID strings"))
|
||||
return
|
||||
}
|
||||
if seen[aid] {
|
||||
reject(apperr.InvalidRequest("duplicate attachment_id"))
|
||||
return
|
||||
}
|
||||
seen[aid] = true
|
||||
}
|
||||
if len(req.AttachmentIDs) == 0 && !validText(req.Text, 4000) {
|
||||
reject(apperr.InvalidRequest("text must be 1..4000 non-space code points when there are no attachments"))
|
||||
return
|
||||
}
|
||||
if !utf8.ValidString(req.Text) || runeLen(req.Text) > 4000 {
|
||||
reject(apperr.InvalidRequest("text must be at most 4000 code points"))
|
||||
return
|
||||
}
|
||||
msg, replayed, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text, req.AttachmentIDs,
|
||||
key, messageFingerprint(req), s.idempotencyTTL)
|
||||
if errors.Is(err, store.ErrIdempotencyConflict) {
|
||||
apperr.Write(w, id, apperr.Conflict("Idempotency-Key was already used with a different request"))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrAttachmentFailed) {
|
||||
apperr.Write(w, id, apperr.Conflict("a failed attachment cannot be attached"))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrAttachmentNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("attachment not found in this chat"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if replayed {
|
||||
w.Header().Set("Idempotent-Replayed", "true")
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, msg)
|
||||
}
|
||||
|
||||
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
chatID, ok := s.chatAccess(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, cur, aerr := pagingParams(r, chatID)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
page := model.Page[*model.Message]{Items: items}
|
||||
if more && len(items) > 0 {
|
||||
last := items[len(items)-1]
|
||||
encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID})
|
||||
page.NextCursor = &encoded
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func (s *Server) notFound(w http.ResponseWriter, r *http.Request) {
|
||||
apperr.Write(w, requestID(r), apperr.NotFound("route not found"))
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
const maxBodyBytes = 1 << 20
|
||||
|
||||
var (
|
||||
usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
)
|
||||
|
||||
func messageFingerprint(req createMessageReq) []byte {
|
||||
b, _ := json.Marshal(req)
|
||||
sum := sha256.Sum256(b)
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
func parseUUID(s string) (string, error) {
|
||||
u, err := uuid.Parse(s)
|
||||
if err != nil || u.String() != s {
|
||||
return "", errors.New("invalid uuid")
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func runeLen(s string) int { return utf8.RuneCountInString(s) }
|
||||
|
||||
func validText(s string, max int) bool {
|
||||
n := runeLen(s)
|
||||
return n >= 1 && n <= max && strings.TrimSpace(s) != ""
|
||||
}
|
||||
|
||||
func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError {
|
||||
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if media != "application/json" {
|
||||
return apperr.UnsupportedMedia("Content-Type must be application/json")
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
|
||||
dec := json.NewDecoder(r.Body)
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(dst); err != nil {
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) {
|
||||
return apperr.PayloadTooLarge("request body too large")
|
||||
}
|
||||
return apperr.InvalidRequest("malformed JSON body")
|
||||
}
|
||||
var extra json.RawMessage
|
||||
if err := dec.Decode(&extra); err != io.EOF {
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) {
|
||||
return apperr.PayloadTooLarge("request body too large")
|
||||
}
|
||||
return apperr.InvalidRequest("malformed JSON body")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) {
|
||||
limit := 50
|
||||
if v := r.URL.Query().Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > 100 {
|
||||
return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100")
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
var cur *store.Cursor
|
||||
if v := r.URL.Query().Get("cursor"); v != "" {
|
||||
c, err := store.DecodeCursor(v)
|
||||
if err == nil {
|
||||
_, err = parseUUID(c.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, nil, apperr.InvalidRequest("invalid cursor")
|
||||
}
|
||||
if c.Scope != scope {
|
||||
return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource")
|
||||
}
|
||||
cur = &c
|
||||
}
|
||||
return limit, cur, nil
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("X-Request-Id", requestID(r))
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func methodNotAllowed(w http.ResponseWriter, r *http.Request) {
|
||||
apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed"))
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const (
|
||||
ctxRequestID ctxKey = iota
|
||||
ctxUser
|
||||
ctxToken
|
||||
)
|
||||
|
||||
func requestIDMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.Header.Get("X-Request-Id")
|
||||
if id == "" {
|
||||
id = uuid.NewString()
|
||||
}
|
||||
w.Header().Set("X-Request-Id", id)
|
||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id)))
|
||||
})
|
||||
}
|
||||
|
||||
func instanceIDMiddleware(instanceID string, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", instanceID)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func requestID(r *http.Request) string {
|
||||
if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" {
|
||||
return id
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
type statusWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
}
|
||||
|
||||
func (s *statusWriter) WriteHeader(code int) {
|
||||
s.status = code
|
||||
s.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
|
||||
next.ServeHTTP(sw, r)
|
||||
|
||||
log.Info("http",
|
||||
"method", r.Method,
|
||||
"path", r.URL.Path,
|
||||
"status", sw.status,
|
||||
"dur_ms", time.Since(start).Milliseconds(),
|
||||
"request_id", requestID(r),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r))
|
||||
apperr.Write(w, requestID(r), apperr.Internal("internal error"))
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func timeoutMiddleware(d, binary time.Duration, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
limit := d
|
||||
if strings.Contains(r.URL.Path, "/attachments") {
|
||||
limit = binary
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), limit)
|
||||
defer cancel()
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
const bearerChallenge = `Bearer realm="messenger"`
|
||||
|
||||
func unauthorizedBearer(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("WWW-Authenticate", bearerChallenge)
|
||||
apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required"))
|
||||
}
|
||||
|
||||
func bearerToken(r *http.Request) (string, bool) {
|
||||
scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||
return "", false
|
||||
}
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" || len(token) > 256 {
|
||||
return "", false
|
||||
}
|
||||
return token, true
|
||||
}
|
||||
|
||||
func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
token, ok := bearerToken(r)
|
||||
if !ok {
|
||||
unauthorizedBearer(w, r)
|
||||
return
|
||||
}
|
||||
user, err := s.sessions.Resolve(r.Context(), token)
|
||||
if errors.Is(err, auth.ErrInvalidSession) {
|
||||
unauthorizedBearer(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
ctx := context.WithValue(r.Context(), ctxUser, user)
|
||||
ctx = context.WithValue(ctx, ctxToken, token)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
func currentUser(r *http.Request) *model.User {
|
||||
return r.Context().Value(ctxUser).(*model.User)
|
||||
}
|
||||
|
||||
func currentToken(r *http.Request) string {
|
||||
return r.Context().Value(ctxToken).(string)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/relay"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
store *store.Store
|
||||
objects *objstore.Store
|
||||
sessions *auth.Sessions
|
||||
hasher *auth.Hasher
|
||||
log *slog.Logger
|
||||
ready *atomic.Bool
|
||||
timeout time.Duration
|
||||
|
||||
instanceID string
|
||||
idempotencyTTL time.Duration
|
||||
|
||||
uploadTimeout time.Duration
|
||||
s3Timeout time.Duration
|
||||
maxUpload int64
|
||||
maxPixels int64
|
||||
uploadSem *semaphore.Weighted
|
||||
|
||||
relay *relay.Relay
|
||||
maxActiveJobs int
|
||||
pipelineVersion int
|
||||
maxAttempts int
|
||||
faultInjection bool
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
InstanceID string
|
||||
IdempotencyTTL time.Duration
|
||||
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
UploadConcurrency int64
|
||||
UploadTimeout time.Duration
|
||||
S3Timeout time.Duration
|
||||
|
||||
Relay *relay.Relay
|
||||
MaxActiveJobs int
|
||||
PipelineVersion int
|
||||
MaxAttempts int
|
||||
FaultInjection bool
|
||||
}
|
||||
|
||||
func NewServer(st *store.Store, objects *objstore.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool, opt Options) *Server {
|
||||
return &Server{store: st, objects: objects, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second,
|
||||
instanceID: opt.InstanceID, idempotencyTTL: opt.IdempotencyTTL,
|
||||
uploadTimeout: opt.UploadTimeout, s3Timeout: opt.S3Timeout,
|
||||
maxUpload: opt.MaxUploadBytes, maxPixels: opt.MaxImagePixels,
|
||||
uploadSem: semaphore.NewWeighted(opt.UploadConcurrency),
|
||||
relay: opt.Relay, maxActiveJobs: opt.MaxActiveJobs, pipelineVersion: opt.PipelineVersion,
|
||||
maxAttempts: opt.MaxAttempts, faultInjection: opt.FaultInjection}
|
||||
}
|
||||
|
||||
func (s *Server) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health/live", s.healthLive)
|
||||
mux.HandleFunc("/health/ready", s.healthReady)
|
||||
mux.HandleFunc("/api/v1/users", s.usersRoute)
|
||||
mux.HandleFunc("/api/v1/auth/sessions", s.createSession)
|
||||
mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession))
|
||||
mux.HandleFunc("/api/v1/users/me", s.auth(s.me))
|
||||
mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute))
|
||||
mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute))
|
||||
mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute))
|
||||
mux.HandleFunc("/api/v1/chats/{chat_id}/attachments", s.auth(s.uploadAttachment))
|
||||
mux.HandleFunc("/api/v1/attachments/{attachment_id}", s.auth(s.getAttachment))
|
||||
mux.HandleFunc("/api/v1/attachments/{attachment_id}/content", s.auth(s.downloadAttachment))
|
||||
mux.HandleFunc("/", s.notFound)
|
||||
|
||||
var h http.Handler = mux
|
||||
h = timeoutMiddleware(s.timeout, s.uploadTimeout, h)
|
||||
h = recoverMiddleware(s.log, h)
|
||||
h = loggingMiddleware(s.log, h)
|
||||
h = requestIDMiddleware(h)
|
||||
h = instanceIDMiddleware(s.instanceID, h)
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package apperr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type Code string
|
||||
|
||||
const (
|
||||
CodeInvalidRequest Code = "invalid_request"
|
||||
CodeUnauthorized Code = "unauthorized"
|
||||
CodeNotFound Code = "not_found"
|
||||
CodeConflict Code = "conflict"
|
||||
CodePayloadTooLarge Code = "payload_too_large"
|
||||
CodeUnsupportedMedia Code = "unsupported_media_type"
|
||||
CodeRateLimited Code = "rate_limited"
|
||||
CodeUnavailable Code = "unavailable"
|
||||
CodeInternal Code = "internal_error"
|
||||
)
|
||||
|
||||
type Body struct {
|
||||
Code Code `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
type Envelope struct {
|
||||
Error Body `json:"error"`
|
||||
}
|
||||
|
||||
type AppError struct {
|
||||
Status int
|
||||
Code Code
|
||||
Msg string
|
||||
}
|
||||
|
||||
func (e *AppError) Error() string { return e.Msg }
|
||||
|
||||
func New(status int, code Code, msg string) *AppError {
|
||||
return &AppError{Status: status, Code: code, Msg: msg}
|
||||
}
|
||||
|
||||
func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) }
|
||||
func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) }
|
||||
func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) }
|
||||
func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) }
|
||||
func PayloadTooLarge(msg string) *AppError {
|
||||
return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg)
|
||||
}
|
||||
func UnsupportedMedia(msg string) *AppError {
|
||||
return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg)
|
||||
}
|
||||
func Unavailable(msg string) *AppError {
|
||||
return New(http.StatusServiceUnavailable, CodeUnavailable, msg)
|
||||
}
|
||||
func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) }
|
||||
|
||||
func Write(w http.ResponseWriter, requestID string, err *AppError) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
w.WriteHeader(err.Status)
|
||||
_ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}})
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
const (
|
||||
argonMemoryKiB = 19 * 1024
|
||||
argonTime = 2
|
||||
argonThreads = 1
|
||||
argonKeyLen = 32
|
||||
argonSaltLen = 16
|
||||
)
|
||||
|
||||
var ErrBadHash = errors.New("malformed password hash")
|
||||
|
||||
type Hasher struct {
|
||||
slots chan struct{}
|
||||
dummy string
|
||||
}
|
||||
|
||||
func NewHasher(concurrency int) *Hasher {
|
||||
h := &Hasher{slots: make(chan struct{}, concurrency)}
|
||||
|
||||
salt := make([]byte, argonSaltLen)
|
||||
_, _ = rand.Read(salt)
|
||||
h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen))
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *Hasher) acquire(ctx context.Context) error {
|
||||
select {
|
||||
case h.slots <- struct{}{}:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Hasher) release() { <-h.slots }
|
||||
|
||||
func encode(salt, key []byte) string {
|
||||
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads,
|
||||
base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key))
|
||||
}
|
||||
|
||||
func (h *Hasher) Hash(ctx context.Context, password string) (string, error) {
|
||||
if err := h.acquire(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer h.release()
|
||||
salt := make([]byte, argonSaltLen)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil
|
||||
}
|
||||
|
||||
func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) {
|
||||
missing := encoded == ""
|
||||
if missing {
|
||||
encoded = h.dummy
|
||||
}
|
||||
parts := strings.Split(encoded, "$")
|
||||
if len(parts) != 6 || parts[1] != "argon2id" {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
var version int
|
||||
var mem uint32
|
||||
var t uint32
|
||||
var p uint8
|
||||
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
if err != nil {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
want, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if err != nil {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
if err := h.acquire(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want)))
|
||||
h.release()
|
||||
return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidSession = errors.New("invalid session")
|
||||
ErrCacheUnavailable = errors.New("session cache unavailable")
|
||||
)
|
||||
|
||||
type SessionCache interface {
|
||||
Get(ctx context.Context, key string) (*CachedSession, error)
|
||||
Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error
|
||||
Del(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
type CachedSession struct {
|
||||
User model.User `json:"user"`
|
||||
ExpiresAt time.Time `json:"exp"`
|
||||
}
|
||||
|
||||
type Sessions struct {
|
||||
store *store.Store
|
||||
cache SessionCache
|
||||
ttl time.Duration
|
||||
cacheMaxTTL time.Duration
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions {
|
||||
return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log}
|
||||
}
|
||||
|
||||
func NewToken() (string, []byte, error) {
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(raw)
|
||||
return token, HashToken(token), nil
|
||||
}
|
||||
|
||||
func HashToken(token string) []byte {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
func cacheKey(hash []byte) string {
|
||||
return "sess:" + base64.RawURLEncoding.EncodeToString(hash)
|
||||
}
|
||||
|
||||
func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) {
|
||||
token, hash, err := NewToken()
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
s.cacheSet(ctx, hash, user, expires)
|
||||
return token, expires, nil
|
||||
}
|
||||
|
||||
func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) {
|
||||
hash := HashToken(token)
|
||||
if s.cache != nil {
|
||||
cs, err := s.cache.Get(ctx, cacheKey(hash))
|
||||
if err == nil && cs != nil {
|
||||
if time.Now().Before(cs.ExpiresAt) {
|
||||
u := cs.User
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
} else if err != nil {
|
||||
s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error())
|
||||
}
|
||||
}
|
||||
user, expires, err := s.store.SessionUser(ctx, hash)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return nil, ErrInvalidSession
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cacheSet(ctx, hash, user, expires)
|
||||
return user, nil
|
||||
}
|
||||
|
||||
func (s *Sessions) Revoke(ctx context.Context, token string) error {
|
||||
hash := HashToken(token)
|
||||
if s.cache != nil {
|
||||
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
|
||||
return ErrCacheUnavailable
|
||||
}
|
||||
}
|
||||
err := s.store.RevokeSession(ctx, hash)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return ErrInvalidSession
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.cache != nil {
|
||||
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
|
||||
|
||||
s.log.Warn("session cache delete after revoke failed", "error", err.Error())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) {
|
||||
if s.cache == nil {
|
||||
return
|
||||
}
|
||||
ttl := time.Until(expires)
|
||||
if ttl > s.cacheMaxTTL {
|
||||
ttl = s.cacheMaxTTL
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return
|
||||
}
|
||||
if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil {
|
||||
s.log.Debug("session cache set failed", "error", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
const (
|
||||
Exchange = "attachments"
|
||||
RoutingKey = "attachment.process.v1"
|
||||
Queue = "attachment.process.v1"
|
||||
DLX = "attachments.dlx"
|
||||
DLQ = "attachment.process.dlq"
|
||||
)
|
||||
|
||||
func Declare(ch *amqp.Channel) error {
|
||||
if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ch.ExchangeDeclare(DLX, "fanout", true, false, false, false, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ch.QueueDeclare(DLQ, true, false, false, false, amqp.Table{"x-queue-type": "quorum"}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ch.QueueBind(DLQ, "", DLX, false, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{
|
||||
"x-queue-type": "quorum",
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-delivery-limit": int32(20),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return ch.QueueBind(Queue, RoutingKey, Exchange, false, nil)
|
||||
}
|
||||
|
||||
func Dial(url string) (*amqp.Connection, error) {
|
||||
return amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Locale: "en_US",
|
||||
Properties: amqp.Table{"connection_name": "messenger"}})
|
||||
}
|
||||
|
||||
type Publisher struct {
|
||||
url string
|
||||
mu sync.Mutex
|
||||
conn *amqp.Connection
|
||||
ch *amqp.Channel
|
||||
}
|
||||
|
||||
func NewPublisher(url string) *Publisher { return &Publisher{url: url} }
|
||||
|
||||
func (p *Publisher) channel() (*amqp.Channel, error) {
|
||||
if p.ch != nil && !p.ch.IsClosed() && p.conn != nil && !p.conn.IsClosed() {
|
||||
return p.ch, nil
|
||||
}
|
||||
p.closeLocked()
|
||||
conn, err := Dial(p.url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := ch.Confirm(false); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := Declare(ch); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
p.conn, p.ch = conn, ch
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (p *Publisher) closeLocked() {
|
||||
if p.conn != nil {
|
||||
_ = p.conn.Close()
|
||||
}
|
||||
p.conn, p.ch = nil, nil
|
||||
}
|
||||
|
||||
func (p *Publisher) Publish(ctx context.Context, messageID string, body []byte) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
ch, err := p.channel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("broker connect: %w", err)
|
||||
}
|
||||
conf, err := ch.PublishWithDeferredConfirmWithContext(ctx, Exchange, RoutingKey, false, false, amqp.Publishing{
|
||||
ContentType: "application/json",
|
||||
DeliveryMode: amqp.Persistent,
|
||||
MessageId: messageID,
|
||||
Type: "attachment.process.v1",
|
||||
Timestamp: time.Now().UTC(),
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
p.closeLocked()
|
||||
return fmt.Errorf("broker publish: %w", err)
|
||||
}
|
||||
ok, err := conf.WaitContext(ctx)
|
||||
if err != nil {
|
||||
p.closeLocked()
|
||||
return fmt.Errorf("broker confirm: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return errors.New("broker nacked publish")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Publisher) Close() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.closeLocked()
|
||||
}
|
||||
Vendored
+80
@@ -0,0 +1,80 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"mtusi-messenger/internal/auth"
|
||||
)
|
||||
|
||||
var ErrOpen = errors.New("cache circuit open")
|
||||
|
||||
type Valkey struct {
|
||||
rdb *redis.Client
|
||||
openUntil atomic.Int64
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewValkey(url string) (*Valkey, error) {
|
||||
opt, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt.DialTimeout = 500 * time.Millisecond
|
||||
opt.ReadTimeout = 300 * time.Millisecond
|
||||
opt.WriteTimeout = 300 * time.Millisecond
|
||||
opt.PoolSize = 10
|
||||
opt.MaxRetries = 0
|
||||
return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil
|
||||
}
|
||||
|
||||
func (v *Valkey) Close() error { return v.rdb.Close() }
|
||||
|
||||
func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) }
|
||||
|
||||
func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() }
|
||||
|
||||
func (v *Valkey) guard(err error) error {
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) {
|
||||
if v.open() {
|
||||
return nil, ErrOpen
|
||||
}
|
||||
raw, err := v.rdb.Get(ctx, key).Bytes()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err := v.guard(err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cs auth.CachedSession
|
||||
if err := json.Unmarshal(raw, &cs); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
return &cs, nil
|
||||
}
|
||||
|
||||
func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error {
|
||||
if v.open() {
|
||||
return ErrOpen
|
||||
}
|
||||
raw, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err())
|
||||
}
|
||||
|
||||
func (v *Valkey) Del(ctx context.Context, key string) error {
|
||||
return v.guard(v.rdb.Del(ctx, key).Err())
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFS embed.FS
|
||||
|
||||
const advisoryLockID = 20260202
|
||||
|
||||
type migration struct {
|
||||
version int
|
||||
name string
|
||||
sql string
|
||||
}
|
||||
|
||||
func loadMigrations() ([]migration, error) {
|
||||
entries, err := fs.ReadDir(migrationFS, "migrations")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []migration
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
prefix, _, ok := strings.Cut(name, "_")
|
||||
if !ok || !strings.HasSuffix(name, ".sql") {
|
||||
return nil, fmt.Errorf("bad migration file name %q", name)
|
||||
}
|
||||
v, err := strconv.Atoi(prefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad migration version in %q", name)
|
||||
}
|
||||
body, err := migrationFS.ReadFile("migrations/" + name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, migration{version: v, name: name, sql: string(body)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
|
||||
for i := 1; i < len(out); i++ {
|
||||
if out[i].version == out[i-1].version {
|
||||
return nil, fmt.Errorf("duplicate migration version %d", out[i].version)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error {
|
||||
migs, err := loadMigrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
|
||||
if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID)
|
||||
|
||||
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version integer PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
applied_at timestamptz NOT NULL DEFAULT now())`); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
applied := map[int]bool{}
|
||||
rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var v int
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
applied[v] = true
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, m := range migs {
|
||||
if applied[m.version] {
|
||||
continue
|
||||
}
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, m.sql); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("migration %s: %w", m.name, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("migration applied", "version", m.version, "name", m.name)
|
||||
}
|
||||
log.Info("migrations up to date", "total", len(migs))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT users_username_key UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32),
|
||||
CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
idem_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
|
||||
DEFERRABLE INITIALLY DEFERRED,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idem_key),
|
||||
CONSTRAINT message_idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT message_idempotency_hash_len CHECK (octet_length(request_hash) = 32)
|
||||
);
|
||||
CREATE INDEX message_idempotency_created_idx ON message_idempotency (created_at);
|
||||
@@ -0,0 +1,44 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
status text NOT NULL,
|
||||
missing_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
updated_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
|
||||
CONSTRAINT attachments_chat_id_key UNIQUE (chat_id, id),
|
||||
CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000),
|
||||
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready'))
|
||||
);
|
||||
CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
|
||||
DEFERRABLE INITIALLY DEFERRED,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_key UNIQUE (message_id, position),
|
||||
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_same_chat_fk FOREIGN KEY (chat_id, attachment_id)
|
||||
REFERENCES attachments (chat_id, id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,75 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text;
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_error_code
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL));
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_object_key_key UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachment_variants_dims CHECK (width > 0 AND height > 0 AND size_bytes > 0)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
status text NOT NULL,
|
||||
attempt integer NOT NULL DEFAULT 1,
|
||||
max_attempts integer NOT NULL,
|
||||
next_attempt_at timestamptz,
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
error_code text,
|
||||
last_error text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND max_attempts >= 1)
|
||||
);
|
||||
CREATE INDEX attachment_jobs_active_idx ON attachment_jobs (status, lease_until)
|
||||
WHERE status IN ('queued', 'processing');
|
||||
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (finished_at) WHERE status = 'failed';
|
||||
|
||||
CREATE TABLE job_outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
|
||||
attempt integer NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
publish_attempts integer NOT NULL DEFAULT 0,
|
||||
last_error text
|
||||
);
|
||||
CREATE INDEX job_outbox_unpublished_idx ON job_outbox (available_at, id) WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
|
||||
attempt integer NOT NULL,
|
||||
event text NOT NULL,
|
||||
error_code text,
|
||||
detail text,
|
||||
actor text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
|
||||
@@ -0,0 +1,31 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type PoolOptions struct {
|
||||
MaxConns int32
|
||||
}
|
||||
|
||||
func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) {
|
||||
cfg, err := pgxpool.ParseConfig(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.MaxConns = opt.MaxConns
|
||||
cfg.MinConns = 1
|
||||
cfg.MaxConnLifetime = 30 * time.Minute
|
||||
cfg.MaxConnIdleTime = 5 * time.Minute
|
||||
cfg.HealthCheckPeriod = 15 * time.Second
|
||||
cfg.ConnConfig.ConnectTimeout = 3 * time.Second
|
||||
rp := cfg.ConnConfig.RuntimeParams
|
||||
rp["application_name"] = "mtusi-messenger"
|
||||
rp["statement_timeout"] = "5000"
|
||||
rp["lock_timeout"] = "2000"
|
||||
rp["idle_in_transaction_session_timeout"] = "10000"
|
||||
return pgxpool.NewWithConfig(ctx, cfg)
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package imaging
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
|
||||
"github.com/gen2brain/webp"
|
||||
"golang.org/x/image/draw"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
const PipelineVersion = 1
|
||||
|
||||
const (
|
||||
ThumbBox = 256
|
||||
CropSide = 128
|
||||
OptimizedBox = 1280
|
||||
JPEGQuality = 85
|
||||
WebPQuality = 80
|
||||
WebPMethod = 4
|
||||
)
|
||||
|
||||
type PermanentError struct{ Code, Detail string }
|
||||
|
||||
func (e *PermanentError) Error() string { return e.Code + ": " + e.Detail }
|
||||
|
||||
func permanent(code, format string, a ...any) error {
|
||||
return &PermanentError{Code: code, Detail: fmt.Sprintf(format, a...)}
|
||||
}
|
||||
|
||||
type Output struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Ext string
|
||||
Data []byte
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
type Limits struct {
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
|
||||
Budget *semaphore.Weighted
|
||||
|
||||
BudgetSize int64
|
||||
}
|
||||
|
||||
func Fit(w, h, box int) (int, int) {
|
||||
if w <= box && h <= box {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
return box, max(1, h*box/w)
|
||||
}
|
||||
return max(1, w*box/h), box
|
||||
}
|
||||
|
||||
func Cost(w, h int) int64 {
|
||||
px := int64(w) * int64(h)
|
||||
return px*4*2 + OptimizedBox*OptimizedBox*4
|
||||
}
|
||||
|
||||
func Process(ctx context.Context, data []byte, contentType string, lim Limits) ([]Output, error) {
|
||||
if int64(len(data)) > lim.MaxBytes {
|
||||
return nil, permanent("image_too_large", "original exceeds %d bytes", lim.MaxBytes)
|
||||
}
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return nil, permanent("invalid_image", "image header is invalid")
|
||||
}
|
||||
if (format == "png") != (contentType == "image/png") || (format == "jpeg") != (contentType == "image/jpeg") {
|
||||
return nil, permanent("invalid_image", "format does not match content type")
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > lim.MaxPixels {
|
||||
return nil, permanent("image_too_large", "image exceeds %d pixels", lim.MaxPixels)
|
||||
}
|
||||
cost := min(Cost(cfg.Width, cfg.Height), lim.BudgetSize)
|
||||
if err := lim.Budget.Acquire(ctx, cost); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type result struct {
|
||||
out []Output
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer lim.Budget.Release(cost)
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
done <- result{err: permanent("invalid_image", "decoder panic")}
|
||||
}
|
||||
}()
|
||||
out, err := build(data, format)
|
||||
done <- result{out, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
return r.out, r.err
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func build(data []byte, format string) ([]Output, error) {
|
||||
var src image.Image
|
||||
var err error
|
||||
orientation := 1
|
||||
if format == "png" {
|
||||
src, err = png.Decode(bytes.NewReader(data))
|
||||
} else {
|
||||
src, err = jpeg.Decode(bytes.NewReader(data))
|
||||
orientation = jpegOrientation(data)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, permanent("invalid_image", "image data is corrupted")
|
||||
}
|
||||
img := flatten(orient(src, orientation))
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
|
||||
var outs []Output
|
||||
tw, th := Fit(w, h, ThumbBox)
|
||||
thumb, err := encodeJPEG(resize(img, b, tw, th))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outs = append(outs, Output{Name: "thumbnail", ContentType: "image/jpeg", Ext: "jpg", Data: thumb, Width: tw, Height: th})
|
||||
|
||||
side := min(w, h)
|
||||
sq := image.Rect(b.Min.X+(w-side)/2, b.Min.Y+(h-side)/2, b.Min.X+(w-side)/2+side, b.Min.Y+(h-side)/2+side)
|
||||
crop, err := encodeJPEG(resize(img, sq, CropSide, CropSide))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outs = append(outs, Output{Name: "crop", ContentType: "image/jpeg", Ext: "jpg", Data: crop, Width: CropSide, Height: CropSide})
|
||||
|
||||
ow, oh := Fit(w, h, OptimizedBox)
|
||||
var opt image.Image = img
|
||||
if ow != w || oh != h {
|
||||
opt = resize(img, b, ow, oh)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := webp.Encode(&buf, opt, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
|
||||
return nil, fmt.Errorf("webp encode: %w", err)
|
||||
}
|
||||
outs = append(outs, Output{Name: "optimized", ContentType: "image/webp", Ext: "webp", Data: buf.Bytes(), Width: ow, Height: oh})
|
||||
return outs, nil
|
||||
}
|
||||
|
||||
func resize(src image.Image, sr image.Rectangle, w, h int) *image.RGBA {
|
||||
dst := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), src, sr, draw.Src, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
func encodeJPEG(img image.Image) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func flatten(src image.Image) image.Image {
|
||||
if op, ok := src.(interface{ Opaque() bool }); ok && op.Opaque() {
|
||||
return src
|
||||
}
|
||||
b := src.Bounds()
|
||||
dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
|
||||
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
||||
draw.Draw(dst, dst.Bounds(), src, b.Min, draw.Over)
|
||||
return dst
|
||||
}
|
||||
|
||||
func orient(src image.Image, o int) image.Image {
|
||||
if o < 2 || o > 8 {
|
||||
return src
|
||||
}
|
||||
b := src.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
dw, dh := w, h
|
||||
if o >= 5 {
|
||||
dw, dh = h, w
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
var dx, dy int
|
||||
switch o {
|
||||
case 2:
|
||||
dx, dy = w-1-x, y
|
||||
case 3:
|
||||
dx, dy = w-1-x, h-1-y
|
||||
case 4:
|
||||
dx, dy = x, h-1-y
|
||||
case 5:
|
||||
dx, dy = y, x
|
||||
case 6:
|
||||
dx, dy = h-1-y, x
|
||||
case 7:
|
||||
dx, dy = h-1-y, w-1-x
|
||||
case 8:
|
||||
dx, dy = y, w-1-x
|
||||
}
|
||||
dst.Set(dx, dy, src.At(b.Min.X+x, b.Min.Y+y))
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
var errNoExif = errors.New("no exif")
|
||||
|
||||
func jpegOrientation(data []byte) int {
|
||||
o, err := parseOrientation(data)
|
||||
if err != nil {
|
||||
return 1
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func parseOrientation(d []byte) (int, error) {
|
||||
if len(d) < 4 || d[0] != 0xff || d[1] != 0xd8 {
|
||||
return 0, errNoExif
|
||||
}
|
||||
i := 2
|
||||
for i+4 <= len(d) {
|
||||
if d[i] != 0xff {
|
||||
return 0, errNoExif
|
||||
}
|
||||
marker := d[i+1]
|
||||
if marker == 0xda || marker == 0xd9 {
|
||||
return 0, errNoExif
|
||||
}
|
||||
l := int(d[i+2])<<8 | int(d[i+3])
|
||||
if l < 2 || i+2+l > len(d) {
|
||||
return 0, errNoExif
|
||||
}
|
||||
seg := d[i+4 : i+2+l]
|
||||
if marker == 0xe1 && len(seg) > 14 && string(seg[:6]) == "Exif\x00\x00" {
|
||||
return tiffOrientation(seg[6:])
|
||||
}
|
||||
i += 2 + l
|
||||
}
|
||||
return 0, errNoExif
|
||||
}
|
||||
|
||||
func tiffOrientation(t []byte) (int, error) {
|
||||
if len(t) < 8 {
|
||||
return 0, errNoExif
|
||||
}
|
||||
var u16 func([]byte) int
|
||||
var u32 func([]byte) int
|
||||
switch string(t[:2]) {
|
||||
case "II":
|
||||
u16 = func(b []byte) int { return int(b[0]) | int(b[1])<<8 }
|
||||
u32 = func(b []byte) int { return int(b[0]) | int(b[1])<<8 | int(b[2])<<16 | int(b[3])<<24 }
|
||||
case "MM":
|
||||
u16 = func(b []byte) int { return int(b[1]) | int(b[0])<<8 }
|
||||
u32 = func(b []byte) int { return int(b[3]) | int(b[2])<<8 | int(b[1])<<16 | int(b[0])<<24 }
|
||||
default:
|
||||
return 0, errNoExif
|
||||
}
|
||||
off := u32(t[4:8])
|
||||
if off < 8 || off+2 > len(t) {
|
||||
return 0, errNoExif
|
||||
}
|
||||
n := u16(t[off : off+2])
|
||||
for k := 0; k < n; k++ {
|
||||
e := off + 2 + k*12
|
||||
if e+12 > len(t) {
|
||||
break
|
||||
}
|
||||
if u16(t[e:e+2]) == 0x0112 {
|
||||
v := u16(t[e+8 : e+10])
|
||||
if v >= 1 && v <= 8 {
|
||||
return v, nil
|
||||
}
|
||||
return 0, errNoExif
|
||||
}
|
||||
}
|
||||
return 0, errNoExif
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package imaging
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
func lim() Limits {
|
||||
return Limits{MaxBytes: 10 << 20, MaxPixels: 40_000_000, Budget: semaphore.NewWeighted(256 << 20), BudgetSize: 256 << 20}
|
||||
}
|
||||
|
||||
func TestFit(t *testing.T) {
|
||||
cases := []struct{ w, h, box, ew, eh int }{
|
||||
{320, 200, 256, 256, 160},
|
||||
{200, 320, 256, 160, 256},
|
||||
{100, 50, 256, 100, 50},
|
||||
{5000, 1, 256, 256, 1},
|
||||
{1, 5000, 1280, 1, 1280},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if w, h := Fit(c.w, c.h, c.box); w != c.ew || h != c.eh {
|
||||
t.Errorf("Fit(%d,%d,%d)=%d,%d want %d,%d", c.w, c.h, c.box, w, h, c.ew, c.eh)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessPNGWithAlpha(t *testing.T) {
|
||||
src := image.NewNRGBA(image.Rect(0, 0, 320, 200))
|
||||
var buf bytes.Buffer
|
||||
_ = png.Encode(&buf, src)
|
||||
outs, err := Process(context.Background(), buf.Bytes(), "image/png", lim())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string][2]int{"thumbnail": {256, 160}, "crop": {128, 128}, "optimized": {320, 200}}
|
||||
for _, o := range outs {
|
||||
if d := want[o.Name]; d != [2]int{o.Width, o.Height} {
|
||||
t.Errorf("%s: %dx%d want %v", o.Name, o.Width, o.Height, d)
|
||||
}
|
||||
if o.Name == "thumbnail" {
|
||||
img, err := jpeg.Decode(bytes.NewReader(o.Data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if img.Bounds().Dx() != 256 || img.Bounds().Dy() != 160 {
|
||||
t.Fatalf("thumbnail real size %v", img.Bounds())
|
||||
}
|
||||
r, g, b, _ := img.At(10, 10).RGBA()
|
||||
if r>>8 < 245 || g>>8 < 245 || b>>8 < 245 {
|
||||
t.Errorf("alpha not composited on white: %d %d %d", r>>8, g>>8, b>>8)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(outs) != 3 {
|
||||
t.Fatalf("got %d variants", len(outs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExifOrientation(t *testing.T) {
|
||||
src := image.NewRGBA(image.Rect(0, 0, 320, 200))
|
||||
for y := 0; y < 200; y++ {
|
||||
for x := 0; x < 320; x++ {
|
||||
src.Set(x, y, color.RGBA{uint8(x), uint8(y), 0, 255})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
_ = jpeg.Encode(&buf, src, nil)
|
||||
data := buf.Bytes()
|
||||
tiff := []byte{'M', 'M', 0, 42, 0, 0, 0, 8, 0, 1, 0x01, 0x12, 0, 3, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0}
|
||||
seg := append([]byte("Exif\x00\x00"), tiff...)
|
||||
app1 := append([]byte{0xff, 0xe1, byte((len(seg) + 2) >> 8), byte(len(seg) + 2)}, seg...)
|
||||
withExif := append(append([]byte{0xff, 0xd8}, app1...), data[2:]...)
|
||||
if o := jpegOrientation(withExif); o != 6 {
|
||||
t.Fatalf("orientation %d", o)
|
||||
}
|
||||
outs, err := Process(context.Background(), withExif, "image/jpeg", lim())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outs[0].Width != 160 || outs[0].Height != 256 {
|
||||
t.Fatalf("thumbnail %dx%d, want 160x256", outs[0].Width, outs[0].Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejects(t *testing.T) {
|
||||
if _, err := Process(context.Background(), []byte("\x89PNG\r\n\x1a\nnot really"), "image/png", lim()); err == nil {
|
||||
t.Fatal("corrupt PNG accepted")
|
||||
} else if pe, ok := err.(*PermanentError); !ok || pe.Code != "invalid_image" {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
}
|
||||
l := lim()
|
||||
l.MaxPixels = 100
|
||||
src := image.NewRGBA(image.Rect(0, 0, 20, 20))
|
||||
var buf bytes.Buffer
|
||||
_ = png.Encode(&buf, src)
|
||||
if _, err := Process(context.Background(), buf.Bytes(), "image/png", l); err == nil {
|
||||
t.Fatal("pixel limit not enforced")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Time struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
func NewTime(t time.Time) Time { return Time{Time: t} }
|
||||
|
||||
func (t Time) MarshalJSON() ([]byte, error) {
|
||||
return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil
|
||||
}
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
|
||||
ObjectKey string `json:"-"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
UpdatedAt Time `json:"updated_at"`
|
||||
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type JobMessage struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
const JobType = "attachment.process.v1"
|
||||
|
||||
type Page[T any] struct {
|
||||
Items []T `json:"items"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type Health struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
Token string `json:"token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresAt Time `json:"expires_at"`
|
||||
User User `json:"user"`
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package objstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
c *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func New(cfg Config) (*Store, error) {
|
||||
u, err := url.Parse(cfg.Endpoint)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return nil, errors.New("S3_ENDPOINT must be http(s)://host:port")
|
||||
}
|
||||
if cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
|
||||
return nil, errors.New("S3 bucket and credentials are required")
|
||||
}
|
||||
|
||||
minio.MaxRetry = 1
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
c, err := minio.New(u.Host, &minio.Options{
|
||||
Transport: transport,
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: u.Scheme == "https",
|
||||
Region: cfg.Region,
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{c: c, bucket: cfg.Bucket}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Bucket() string { return s.bucket }
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
ok, err := s.c.BucketExists(ctx, s.bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) EnsureBucket(ctx context.Context, region string) (bool, error) {
|
||||
ok, err := s.c.BucketExists(ctx, s.bucket)
|
||||
if err != nil || ok {
|
||||
return false, err
|
||||
}
|
||||
if err := s.c.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: region}); err != nil {
|
||||
if code := minio.ToErrorResponse(err).Code; code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
_, err := s.c.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType, SendContentMd5: true, DisableMultipart: true})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return obj, info.Size, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
|
||||
info, err := s.c.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return Object{}, err
|
||||
}
|
||||
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Remove(ctx context.Context, key string) error {
|
||||
return s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (s *Store) List(ctx context.Context, prefix string, fn func(Object) error) error {
|
||||
for info := range s.c.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||
if info.Err != nil {
|
||||
return info.Err
|
||||
}
|
||||
if err := fn(Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch minio.ToErrorResponse(err).Code {
|
||||
case "NoSuchKey", "NotFound":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Store) ReadAll(ctx context.Context, key string, limit int64) ([]byte, error) {
|
||||
obj, size, err := s.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer obj.Close()
|
||||
if size > limit {
|
||||
return nil, fmt.Errorf("object larger than %d bytes", limit)
|
||||
}
|
||||
buf := bytes.NewBuffer(make([]byte, 0, size))
|
||||
if _, err := io.Copy(buf, io.LimitReader(obj, limit+1)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package relay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/internal/broker"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
type Relay struct {
|
||||
st *store.Store
|
||||
pub *broker.Publisher
|
||||
log *slog.Logger
|
||||
kick chan struct{}
|
||||
every time.Duration
|
||||
failed bool
|
||||
}
|
||||
|
||||
func New(st *store.Store, pub *broker.Publisher, log *slog.Logger, every time.Duration) *Relay {
|
||||
return &Relay{st: st, pub: pub, log: log, kick: make(chan struct{}, 1), every: every}
|
||||
}
|
||||
|
||||
func (r *Relay) Kick() {
|
||||
select {
|
||||
case r.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Relay) Run(ctx context.Context) {
|
||||
t := time.NewTicker(r.every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
r.drain(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
case <-r.kick:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Relay) drain(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
pctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
n, err := r.st.RelayOutbox(pctx, 50, func(row store.OutboxRow) error {
|
||||
c, cc := context.WithTimeout(pctx, 5*time.Second)
|
||||
defer cc()
|
||||
return r.pub.Publish(c, row.JobID, row.Payload)
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
if !r.failed {
|
||||
r.log.Warn("outbox relay pass failed", "error", err.Error())
|
||||
}
|
||||
r.failed = true
|
||||
return
|
||||
}
|
||||
if r.failed {
|
||||
r.log.Info("outbox relay recovered")
|
||||
r.failed = false
|
||||
}
|
||||
if n > 0 {
|
||||
r.log.Debug("outbox published", "count", n)
|
||||
}
|
||||
if n < 50 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
const attachmentColumns = `a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.object_key, a.status, a.error_code, a.pipeline_version, a.created_at, a.updated_at`
|
||||
|
||||
func scanAttachment(row pgx.Row) (*model.Attachment, error) {
|
||||
a := &model.Attachment{Variants: []model.Variant{}}
|
||||
var created, updated time.Time
|
||||
err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.ObjectKey, &a.Status, &a.ErrorCode, &a.PipelineVersion, &created, &updated)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.CreatedAt, a.UpdatedAt = utc(created), utc(updated)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *Store) InsertPendingAttachment(ctx context.Context, a *model.Attachment, width, height int, stagingKey string) error {
|
||||
var created time.Time
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
width, height, object_key, staging_key, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'pending')
|
||||
RETURNING created_at`,
|
||||
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
|
||||
width, height, a.ObjectKey, stagingKey).Scan(&created)
|
||||
if pgCode(err) == "23503" {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.CreatedAt = utc(created)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) DeletePendingAttachment(ctx context.Context, id string) (bool, error) {
|
||||
tag, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentForUser(ctx context.Context, id, userID string) (*model.Attachment, error) {
|
||||
a, err := scanAttachment(s.pool.QueryRow(ctx, `
|
||||
SELECT `+attachmentColumns+`
|
||||
FROM attachments a
|
||||
WHERE a.id = $1 AND a.status <> 'pending'
|
||||
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2)`,
|
||||
id, userID))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Status != "ready" {
|
||||
return a, nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END`, a.ID, a.PipelineVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v model.Variant
|
||||
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.Variants = append(a.Variants, v)
|
||||
}
|
||||
return a, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Cursor struct {
|
||||
Scope string `json:"s"`
|
||||
TS time.Time `json:"t"`
|
||||
ID string `json:"i"`
|
||||
}
|
||||
|
||||
func EncodeCursor(c Cursor) string {
|
||||
b, _ := json.Marshal(c)
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
func DecodeCursor(s string) (Cursor, error) {
|
||||
var c Cursor
|
||||
b, err := base64.RawURLEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
return c, ErrInvalidCursor
|
||||
}
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
return c, ErrInvalidCursor
|
||||
}
|
||||
if c.Scope == "" || c.ID == "" || c.TS.IsZero() {
|
||||
return c, ErrInvalidCursor
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrJobNotFailed = errors.New("job is not in failed state")
|
||||
ErrJobAlreadyOK = errors.New("job already completed")
|
||||
ErrStagingGone = errors.New("staging original already removed by retention")
|
||||
ErrQueueFull = errors.New("too much unfinished work")
|
||||
)
|
||||
|
||||
func (s *Store) inTx(ctx context.Context, fn func(pgx.Tx) error) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
if err := fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func jobPayload(jobID, attID string, pv, attempt int, created time.Time) []byte {
|
||||
b, _ := json.Marshal(model.JobMessage{
|
||||
SchemaVersion: 1, JobID: jobID, Type: model.JobType, AttachmentID: attID,
|
||||
PipelineVersion: pv, Attempt: attempt, CreatedAt: created.UTC().Format(time.RFC3339Nano),
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func addOutbox(ctx context.Context, tx pgx.Tx, jobID, attID string, pv, attempt int, availableAt *time.Time) error {
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO job_outbox (job_id, attempt, payload, available_at)
|
||||
VALUES ($1, $2, $3, COALESCE($4, clock_timestamp()))`,
|
||||
jobID, attempt, jobPayload(jobID, attID, pv, attempt, time.Now()), availableAt)
|
||||
return err
|
||||
}
|
||||
|
||||
func addEvent(ctx context.Context, tx pgx.Tx, jobID string, attempt int, event string, code, detail *string, actor string) error {
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO job_events (job_id, attempt, event, error_code, detail, actor)
|
||||
VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''))`, jobID, attempt, event, code, detail, actor)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) QueueAttachment(ctx context.Context, attID string, pipelineVersion, maxAttempts, maxActive int) (string, time.Time, error) {
|
||||
jobID := uuid.NewString()
|
||||
var updated time.Time
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
if maxActive > 0 {
|
||||
var active int
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(20260505)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')`).Scan(&active); err != nil {
|
||||
return err
|
||||
}
|
||||
if active >= maxActive {
|
||||
return ErrQueueFull
|
||||
}
|
||||
}
|
||||
err := tx.QueryRow(ctx, `
|
||||
UPDATE attachments SET status = 'queued', pipeline_version = $2,
|
||||
updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING updated_at`, attID, pipelineVersion).Scan(&updated)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts)
|
||||
VALUES ($1, $2, $3, 'queued', 1, $4)`, jobID, attID, pipelineVersion, maxAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addEvent(ctx, tx, jobID, 1, "queued", nil, nil, "api"); err != nil {
|
||||
return err
|
||||
}
|
||||
return addOutbox(ctx, tx, jobID, attID, pipelineVersion, 1, nil)
|
||||
})
|
||||
return jobID, updated.UTC(), err
|
||||
}
|
||||
|
||||
func (s *Store) ActiveJobs(ctx context.Context) (int, error) {
|
||||
var n int
|
||||
err := s.pool.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
type OutboxRow struct {
|
||||
ID int64
|
||||
JobID string
|
||||
Attempt int
|
||||
Payload []byte
|
||||
}
|
||||
|
||||
func (s *Store) RelayOutbox(ctx context.Context, limit int, publish func(OutboxRow) error) (int, error) {
|
||||
published := 0
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT id, job_id, attempt, payload FROM job_outbox
|
||||
WHERE published_at IS NULL AND available_at <= clock_timestamp()
|
||||
ORDER BY available_at, id
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED`, limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var batch []OutboxRow
|
||||
for rows.Next() {
|
||||
var r OutboxRow
|
||||
if err := rows.Scan(&r.ID, &r.JobID, &r.Attempt, &r.Payload); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
batch = append(batch, r)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range batch {
|
||||
if perr := publish(r); perr != nil {
|
||||
msg := perr.Error()
|
||||
_, _ = tx.Exec(ctx, `UPDATE job_outbox SET publish_attempts = publish_attempts + 1, last_error = $2 WHERE id = $1`, r.ID, msg)
|
||||
return nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE job_outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1, last_error = NULL
|
||||
WHERE id = $1`, r.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
published++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return published, nil
|
||||
}
|
||||
|
||||
func (s *Store) PurgeOutbox(ctx context.Context, olderThan time.Duration) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx, `DELETE FROM job_outbox WHERE published_at < now() - make_interval(secs => $1)`, olderThan.Seconds())
|
||||
return tag.RowsAffected(), err
|
||||
}
|
||||
|
||||
type JobWork struct {
|
||||
JobID string
|
||||
AttachmentID string
|
||||
ChatID string
|
||||
Filename string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
ObjectKey string
|
||||
StagingKey string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
MaxAttempts int
|
||||
}
|
||||
|
||||
type ClaimResult string
|
||||
|
||||
const (
|
||||
Claimed ClaimResult = "claimed"
|
||||
Done ClaimResult = "done"
|
||||
Stale ClaimResult = "stale"
|
||||
Busy ClaimResult = "busy"
|
||||
NotFound ClaimResult = "notfound"
|
||||
NotYet ClaimResult = "notyet"
|
||||
)
|
||||
|
||||
func (s *Store) ClaimJob(ctx context.Context, jobID string, attempt int, owner string, lease time.Duration, takeover bool) (*JobWork, ClaimResult, error) {
|
||||
var w JobWork
|
||||
result := Claimed
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
var status string
|
||||
var curAttempt int
|
||||
var next *time.Time
|
||||
err := tx.QueryRow(ctx, `SELECT status, attempt, next_attempt_at FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID).
|
||||
Scan(&status, &curAttempt, &next)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
result = NotFound
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case status == "ready" || status == "failed":
|
||||
result = Done
|
||||
return nil
|
||||
case curAttempt != attempt:
|
||||
result = Stale
|
||||
return nil
|
||||
case status == "processing" && !takeover:
|
||||
var expired bool
|
||||
if err := tx.QueryRow(ctx, `SELECT lease_until < clock_timestamp() FROM attachment_jobs WHERE job_id = $1`, jobID).Scan(&expired); err != nil {
|
||||
return err
|
||||
}
|
||||
if !expired {
|
||||
result = Busy
|
||||
return nil
|
||||
}
|
||||
case status == "queued" && next != nil && next.After(time.Now().Add(2*time.Second)):
|
||||
result = NotYet
|
||||
return nil
|
||||
}
|
||||
err = tx.QueryRow(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing', lease_owner = $2,
|
||||
lease_until = clock_timestamp() + make_interval(secs => $3),
|
||||
started_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE job_id = $1
|
||||
RETURNING attachment_id, pipeline_version, attempt, max_attempts`, jobID, owner, lease.Seconds()).
|
||||
Scan(&w.AttachmentID, &w.PipelineVersion, &w.Attempt, &w.MaxAttempts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = tx.QueryRow(ctx, `
|
||||
UPDATE attachments SET status = 'processing', updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status IN ('queued', 'processing')
|
||||
RETURNING chat_id, filename, content_type, size_bytes, sha256, object_key, COALESCE(staging_key, '')`, w.AttachmentID).
|
||||
Scan(&w.ChatID, &w.Filename, &w.ContentType, &w.SizeBytes, &w.SHA256, &w.ObjectKey, &w.StagingKey)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
result = NotFound
|
||||
return errors.New("rollback")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ev := "started"
|
||||
if status == "processing" {
|
||||
ev = "taken_over"
|
||||
}
|
||||
return addEvent(ctx, tx, jobID, attempt, ev, nil, nil, owner)
|
||||
})
|
||||
if result == NotFound {
|
||||
return nil, NotFound, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if result != Claimed {
|
||||
return nil, result, nil
|
||||
}
|
||||
w.JobID = jobID
|
||||
return &w, Claimed, nil
|
||||
}
|
||||
|
||||
func (s *Store) ExtendLease(ctx context.Context, jobID string, attempt int, owner string, lease time.Duration) (bool, error) {
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $4)
|
||||
WHERE job_id = $1 AND attempt = $2 AND lease_owner = $3 AND status = 'processing'`,
|
||||
jobID, attempt, owner, lease.Seconds())
|
||||
return tag.RowsAffected() == 1, err
|
||||
}
|
||||
|
||||
func (s *Store) CompleteJob(ctx context.Context, w *JobWork, variants []model.Variant) (bool, error) {
|
||||
ok := false
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', lease_owner = NULL, lease_until = NULL, error_code = NULL,
|
||||
finished_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE job_id = $1 AND attempt = $2 AND status = 'processing'`, w.JobID, w.Attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return nil
|
||||
}
|
||||
for _, v := range variants {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type,
|
||||
size_bytes, width, height, sha256, object_key)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING`,
|
||||
w.AttachmentID, w.PipelineVersion, v.Name, v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256, v.ObjectKey); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status = 'processing'`, w.AttachmentID); err != nil {
|
||||
return err
|
||||
}
|
||||
ok = true
|
||||
return addEvent(ctx, tx, w.JobID, w.Attempt, "ready", nil, nil, "")
|
||||
})
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (s *Store) ClearStaging(ctx context.Context, attID string) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1 AND status IN ('ready', 'failed')`, attID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) FailOrRetry(ctx context.Context, jobID string, attempt int, code, detail string, permanent bool, backoff time.Duration, actor string) (string, error) {
|
||||
out := ""
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
var attID string
|
||||
var pv, maxAttempts int
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT attachment_id, pipeline_version, max_attempts FROM attachment_jobs
|
||||
WHERE job_id = $1 AND attempt = $2 AND status = 'processing' FOR UPDATE`, jobID, attempt).
|
||||
Scan(&attID, &pv, &maxAttempts)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return failOrRetryLocked(ctx, tx, jobID, attID, pv, attempt, maxAttempts, code, detail, permanent, backoff, actor, &out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func failOrRetryLocked(ctx context.Context, tx pgx.Tx, jobID, attID string, pv, attempt, maxAttempts int, code, detail string, permanent bool, backoff time.Duration, actor string, out *string) error {
|
||||
if !permanent && attempt >= maxAttempts {
|
||||
permanent = true
|
||||
detail = code + ": " + detail
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
if permanent {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs SET status = 'failed', error_code = $2, last_error = $3,
|
||||
lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE job_id = $1`, jobID, code, detail); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachments SET status = 'failed', error_code = $2, updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status IN ('queued', 'processing')`, attID, code); err != nil {
|
||||
return err
|
||||
}
|
||||
*out = "failed"
|
||||
return addEvent(ctx, tx, jobID, attempt, "failed", &code, &detail, actor)
|
||||
}
|
||||
next := time.Now().Add(backoff)
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs SET status = 'queued', attempt = attempt + 1, next_attempt_at = $2,
|
||||
error_code = $3, last_error = $4, lease_owner = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE job_id = $1`, jobID, next, code, detail); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachments SET status = 'queued', updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status IN ('queued', 'processing')`, attID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addEvent(ctx, tx, jobID, attempt, "retry_scheduled", &code, &detail, actor); err != nil {
|
||||
return err
|
||||
}
|
||||
*out = "queued"
|
||||
return addOutbox(ctx, tx, jobID, attID, pv, attempt+1, &next)
|
||||
}
|
||||
|
||||
func (s *Store) RecoverExpiredLeases(ctx context.Context, limit int) (int, error) {
|
||||
n := 0
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts FROM attachment_jobs
|
||||
WHERE status = 'processing' AND lease_until < clock_timestamp()
|
||||
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED`, limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type rec struct {
|
||||
job, att string
|
||||
pv, attempt, maxAt int
|
||||
}
|
||||
var list []rec
|
||||
for rows.Next() {
|
||||
var r rec
|
||||
if err := rows.Scan(&r.job, &r.att, &r.pv, &r.attempt, &r.maxAt); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
list = append(list, r)
|
||||
}
|
||||
rows.Close()
|
||||
for _, r := range list {
|
||||
var out string
|
||||
if err := failOrRetryLocked(ctx, tx, r.job, r.att, r.pv, r.attempt, r.maxAt, "lease_expired",
|
||||
"worker lease expired", false, 0, "sweeper", &out); err != nil {
|
||||
return err
|
||||
}
|
||||
n++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (s *Store) ReplayFailed(ctx context.Context, attID, actor string, extraAttempts int) (string, int, error) {
|
||||
var jobID string
|
||||
var attempt int
|
||||
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
||||
var status string
|
||||
var pv int
|
||||
var staging *string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT j.job_id, j.status, j.attempt, j.pipeline_version, a.staging_key
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.attachment_id = $1 ORDER BY j.pipeline_version DESC LIMIT 1
|
||||
FOR UPDATE`, attID).Scan(&jobID, &status, &attempt, &pv, &staging)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch status {
|
||||
case "ready":
|
||||
return ErrJobAlreadyOK
|
||||
case "failed":
|
||||
default:
|
||||
return ErrJobNotFailed
|
||||
}
|
||||
if staging == nil {
|
||||
return ErrStagingGone
|
||||
}
|
||||
attempt++
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs SET status = 'queued', attempt = $2, max_attempts = $3, next_attempt_at = NULL,
|
||||
error_code = NULL, finished_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE job_id = $1`, jobID, attempt, attempt+extraAttempts-1); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status = 'failed'`, attID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addEvent(ctx, tx, jobID, attempt, "replay", nil, nil, actor); err != nil {
|
||||
return err
|
||||
}
|
||||
return addOutbox(ctx, tx, jobID, attID, pv, attempt, nil)
|
||||
})
|
||||
return jobID, attempt, err
|
||||
}
|
||||
|
||||
type StagingRef struct {
|
||||
AttachmentID string
|
||||
StagingKey string
|
||||
Status string
|
||||
}
|
||||
|
||||
func (s *Store) StaleStaging(ctx context.Context, pendingGrace, failedRetention time.Duration, limit int) ([]StagingRef, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, staging_key, status FROM attachments
|
||||
WHERE staging_key IS NOT NULL
|
||||
AND ((status = 'pending' AND created_at < now() - make_interval(secs => $1))
|
||||
OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
|
||||
LIMIT $3`, pendingGrace.Seconds(), failedRetention.Seconds(), limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []StagingRef
|
||||
for rows.Next() {
|
||||
var r StagingRef
|
||||
if err := rows.Scan(&r.AttachmentID, &r.StagingKey, &r.Status); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ForgetStaging(ctx context.Context, r StagingRef) error {
|
||||
if r.Status == "pending" {
|
||||
_, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, r.AttachmentID)
|
||||
return err
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1 AND status = 'failed'`, r.AttachmentID)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrUsernameTaken = errors.New("username taken")
|
||||
ErrUnknownMember = errors.New("unknown member")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
|
||||
|
||||
func IsUnavailable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) {
|
||||
return true
|
||||
}
|
||||
var connErr *pgconn.ConnectError
|
||||
if errors.As(err, &connErr) {
|
||||
return true
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
|
||||
switch {
|
||||
case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03",
|
||||
len(pgErr.Code) == 5 && pgErr.Code[:2] == "08":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed)
|
||||
}
|
||||
|
||||
func pgCode(err error) string {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
return pgErr.Code
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) }
|
||||
|
||||
func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) {
|
||||
u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName}
|
||||
var created time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ($1, $2, $3, $4) RETURNING created_at`,
|
||||
u.ID, username, displayName, passwordHash).Scan(&created)
|
||||
if pgCode(err) == "23505" {
|
||||
return nil, ErrUsernameTaken
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u.CreatedAt = utc(created)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) {
|
||||
u := &model.User{}
|
||||
var created time.Time
|
||||
var hash string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`,
|
||||
username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
u.CreatedAt = utc(created)
|
||||
return u, hash, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) {
|
||||
var expires time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3))
|
||||
RETURNING expires_at`,
|
||||
tokenHash, userID, ttl.Seconds()).Scan(&expires)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return expires.UTC(), nil
|
||||
}
|
||||
|
||||
func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) {
|
||||
u := &model.User{}
|
||||
var created, expires time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`,
|
||||
tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, time.Time{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
u.CreatedAt = utc(created)
|
||||
return u, expires.UTC(), nil
|
||||
}
|
||||
|
||||
func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`UPDATE sessions SET revoked_at = now()
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM sessions
|
||||
WHERE expires_at < now() - make_interval(secs => $1)
|
||||
OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) {
|
||||
c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)}
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
var created time.Time
|
||||
if err := tx.QueryRow(ctx,
|
||||
`INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`,
|
||||
c.ID, title, creatorID).Scan(&created); err != nil {
|
||||
return err
|
||||
}
|
||||
c.CreatedAt = utc(created)
|
||||
_, err := tx.Exec(ctx,
|
||||
`INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT $1, m.user_id, m.ord - 1
|
||||
FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`,
|
||||
c.ID, memberIDs)
|
||||
return err
|
||||
})
|
||||
if pgCode(err) == "23503" {
|
||||
return nil, ErrUnknownMember
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) {
|
||||
var ok bool
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`,
|
||||
chatID, userID).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) {
|
||||
c := &model.Chat{}
|
||||
var created time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT c.id, c.title, c.created_at,
|
||||
array_agg(m.user_id::text ORDER BY m.position)
|
||||
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE c.id = $1
|
||||
AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2)
|
||||
GROUP BY c.id`,
|
||||
chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.CreatedAt = utc(created)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) {
|
||||
var ts *time.Time
|
||||
var cid *string
|
||||
if cur != nil {
|
||||
ts, cid = &cur.TS, &cur.ID
|
||||
}
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT c.id, c.title, c.created_at,
|
||||
(SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id)
|
||||
FROM chat_members me JOIN chats c ON c.id = me.chat_id
|
||||
WHERE me.user_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid))
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $4`,
|
||||
userID, ts, cid, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]*model.Chat, 0, limit+1)
|
||||
for rows.Next() {
|
||||
c := &model.Chat{}
|
||||
var created time.Time
|
||||
if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c.CreatedAt = utc(created)
|
||||
out = append(out, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(out) > limit
|
||||
if more {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, more, nil
|
||||
}
|
||||
|
||||
var (
|
||||
ErrIdempotencyConflict = errors.New("idempotency key reused with different body")
|
||||
ErrAttachmentNotFound = errors.New("attachment not found in chat")
|
||||
ErrAttachmentFailed = errors.New("attachment processing failed")
|
||||
)
|
||||
|
||||
func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, key string, reqHash []byte, ttl time.Duration) (*model.Message, bool, error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
newID := uuid.NewString()
|
||||
var member, attachmentsOK bool
|
||||
var failedCount int64
|
||||
var created *time.Time
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
WITH member AS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1
|
||||
), att AS (
|
||||
SELECT count(*) FILTER (WHERE status IN ('queued', 'processing', 'ready')) = cardinality($8::uuid[]) AS ok,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM attachments
|
||||
WHERE chat_id = $2 AND id = ANY($8::uuid[])
|
||||
), k AS (
|
||||
INSERT INTO message_idempotency (user_id, chat_id, idem_key, request_hash, message_id)
|
||||
SELECT $1, $2, $3, $4, $5 FROM member, att WHERE att.ok
|
||||
ON CONFLICT (user_id, chat_id, idem_key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = now()
|
||||
WHERE message_idempotency.created_at < now() - make_interval(secs => $7)
|
||||
RETURNING message_id
|
||||
), m AS (
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
SELECT $5, $2, $1, $6 FROM k
|
||||
RETURNING created_at
|
||||
), ma AS (
|
||||
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
SELECT $5, $2, a.id, a.ord - 1
|
||||
FROM unnest($8::uuid[]) WITH ORDINALITY AS a(id, ord)
|
||||
WHERE EXISTS (SELECT 1 FROM m)
|
||||
)
|
||||
SELECT EXISTS (SELECT 1 FROM member), (SELECT ok FROM att), (SELECT failed FROM att), (SELECT created_at FROM m)`,
|
||||
senderID, chatID, key, reqHash, newID, text, ttl.Seconds(), attachmentIDs).Scan(&member, &attachmentsOK, &failedCount, &created)
|
||||
if pgCode(err) == "23503" {
|
||||
return nil, false, ErrAttachmentNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !member {
|
||||
return nil, false, ErrNotFound
|
||||
}
|
||||
if !attachmentsOK {
|
||||
if failedCount > 0 {
|
||||
return nil, false, ErrAttachmentFailed
|
||||
}
|
||||
return nil, false, ErrAttachmentNotFound
|
||||
}
|
||||
if created != nil {
|
||||
return &model.Message{ID: newID, ChatID: chatID, SenderID: senderID, Text: text,
|
||||
CreatedAt: utc(*created), AttachmentIDs: attachmentIDs}, false, nil
|
||||
}
|
||||
var stored []byte
|
||||
var at time.Time
|
||||
m := &model.Message{ChatID: chatID}
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
SELECT i.request_hash, m.id, m.sender_id, m.text, m.created_at,
|
||||
`+attachmentIDsSQL+`
|
||||
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
|
||||
WHERE i.user_id = $1 AND i.chat_id = $2 AND i.idem_key = $3
|
||||
AND i.created_at >= now() - make_interval(secs => $4)`,
|
||||
senderID, chatID, key, ttl.Seconds()).Scan(&stored, &m.ID, &m.SenderID, &m.Text, &at, &m.AttachmentIDs)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !bytes.Equal(stored, reqHash) {
|
||||
return nil, false, ErrIdempotencyConflict
|
||||
}
|
||||
m.CreatedAt = utc(at)
|
||||
return m, true, nil
|
||||
}
|
||||
return nil, false, errors.New("idempotency record kept changing")
|
||||
}
|
||||
|
||||
const attachmentIDsSQL = `COALESCE((SELECT array_agg(ma.attachment_id::text ORDER BY ma.position)
|
||||
FROM message_attachments ma WHERE ma.message_id = m.id), '{}')`
|
||||
|
||||
func (s *Store) PurgeIdempotency(ctx context.Context, ttl time.Duration) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM message_idempotency WHERE created_at < now() - make_interval(secs => $1)`, ttl.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) {
|
||||
var (
|
||||
rows pgx.Rows
|
||||
err error
|
||||
)
|
||||
if cur == nil {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1
|
||||
ORDER BY m.created_at, m.id LIMIT $2`, chatID, limit+1)
|
||||
} else {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3::uuid)
|
||||
ORDER BY m.created_at, m.id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]*model.Message, 0, limit+1)
|
||||
for rows.Next() {
|
||||
m := &model.Message{}
|
||||
var created time.Time
|
||||
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
m.CreatedAt = utc(created)
|
||||
out = append(out, m)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(out) > limit
|
||||
if more {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, more, nil
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"mtusi-messenger/internal/broker"
|
||||
"mtusi-messenger/internal/imaging"
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
BrokerURL string
|
||||
Concurrency int
|
||||
MaxAttempts int
|
||||
JobTimeout time.Duration
|
||||
Lease time.Duration
|
||||
BackoffBase time.Duration
|
||||
BackoffMax time.Duration
|
||||
StagingRetain time.Duration
|
||||
PendingGrace time.Duration
|
||||
FaultInjection bool
|
||||
Limits imaging.Limits
|
||||
SweepEvery time.Duration
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
cfg Config
|
||||
st *store.Store
|
||||
s3 *objstore.Store
|
||||
log *slog.Logger
|
||||
owner string
|
||||
inFly sync.WaitGroup
|
||||
}
|
||||
|
||||
func New(cfg Config, st *store.Store, s3 *objstore.Store, log *slog.Logger) *Worker {
|
||||
host, _ := os.Hostname()
|
||||
return &Worker{cfg: cfg, st: st, s3: s3, log: log, owner: host + "/" + uuid.NewString()[:8]}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) error {
|
||||
go w.sweeper(ctx)
|
||||
for ctx.Err() == nil {
|
||||
if err := w.consume(ctx); err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("consumer stopped, reconnecting", "error", err.Error())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
w.inFly.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) consume(ctx context.Context) error {
|
||||
conn, err := broker.Dial(w.cfg.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.Declare(ch); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ch.Qos(w.cfg.Concurrency, 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
deliveries, err := ch.ConsumeWithContext(ctx, broker.Queue, w.owner, false, false, false, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.log.Info("worker consuming", "queue", broker.Queue, "concurrency", w.cfg.Concurrency, "owner", w.owner)
|
||||
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < w.cfg.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for d := range deliveries {
|
||||
w.inFly.Add(1)
|
||||
w.handle(ctx, d)
|
||||
w.inFly.Done()
|
||||
}
|
||||
}()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = ch.Cancel(w.owner, false)
|
||||
wg.Wait()
|
||||
return nil
|
||||
case e := <-closed:
|
||||
wg.Wait()
|
||||
if e == nil {
|
||||
return errors.New("broker connection closed")
|
||||
}
|
||||
return e
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) fault(job *store.JobWork, point string) bool {
|
||||
return w.cfg.FaultInjection && strings.HasPrefix(job.Filename, "fault-"+point)
|
||||
}
|
||||
|
||||
func (w *Worker) handle(ctx context.Context, d amqp.Delivery) {
|
||||
var msg model.JobMessage
|
||||
if err := json.Unmarshal(d.Body, &msg); err != nil || msg.SchemaVersion != 1 || msg.Type != model.JobType ||
|
||||
uuid.Validate(msg.JobID) != nil || uuid.Validate(msg.AttachmentID) != nil || msg.Attempt < 1 || msg.PipelineVersion < 1 {
|
||||
w.log.Error("malformed job message -> DLQ", "message_id", d.MessageId)
|
||||
_ = d.Nack(false, false)
|
||||
return
|
||||
}
|
||||
log := w.log.With("job_id", msg.JobID, "attachment_id", msg.AttachmentID, "attempt", msg.Attempt, "redelivered", d.Redelivered)
|
||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
job, res, err := w.st.ClaimJob(cctx, msg.JobID, msg.Attempt, w.owner, w.cfg.Lease, d.Redelivered)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Warn("claim failed, requeue", "error", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
_ = d.Nack(false, true)
|
||||
return
|
||||
}
|
||||
if res != store.Claimed {
|
||||
|
||||
log.Info("job delivery is a no-op", "result", string(res))
|
||||
_ = d.Ack(false)
|
||||
return
|
||||
}
|
||||
if job.AttachmentID != msg.AttachmentID {
|
||||
log.Error("message attachment_id does not match job")
|
||||
_ = d.Ack(false)
|
||||
return
|
||||
}
|
||||
|
||||
if w.fault(job, "hang") && job.Attempt == 1 {
|
||||
|
||||
log.Warn("FAULT: hang without heartbeat", "for", (3 * w.cfg.Lease).String())
|
||||
time.Sleep(3 * w.cfg.Lease)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
variants, perr := w.process(ctx, job, d.Redelivered, log)
|
||||
if perr == nil {
|
||||
cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
ok, err := w.st.CompleteJob(cctx, job, variants)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Warn("result commit failed, requeue", "error", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
_ = d.Nack(false, true)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
log.Info("stale attempt finished after takeover; result already owned by newer attempt")
|
||||
} else {
|
||||
log.Info("job ready", "ms", time.Since(start).Milliseconds())
|
||||
w.dropStaging(job, log)
|
||||
}
|
||||
if w.fault(job, "worker-before-ack") && !d.Redelivered {
|
||||
log.Warn("FAULT: exit after COMMIT before ack")
|
||||
os.Exit(42)
|
||||
}
|
||||
_ = d.Ack(false)
|
||||
return
|
||||
}
|
||||
|
||||
code, permanent := classify(perr)
|
||||
backoff := w.backoff(job.Attempt)
|
||||
cctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
|
||||
status, err := w.st.FailOrRetry(cctx, job.JobID, job.Attempt, code, safeDetail(perr), permanent, backoff, w.owner)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Warn("failure bookkeeping failed, requeue", "error", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
_ = d.Nack(false, true)
|
||||
return
|
||||
}
|
||||
log.Warn("job attempt failed", "error_code", code, "permanent", permanent, "next", status, "backoff_ms", backoff.Milliseconds())
|
||||
_ = d.Ack(false)
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, job *store.JobWork, redelivered bool, log *slog.Logger) ([]model.Variant, error) {
|
||||
pctx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
|
||||
defer cancel()
|
||||
|
||||
hbDone := make(chan struct{})
|
||||
defer close(hbDone)
|
||||
go func() {
|
||||
t := time.NewTicker(w.cfg.Lease / 3)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hbDone:
|
||||
return
|
||||
case <-pctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
c, cc := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
_, _ = w.st.ExtendLease(c, job.JobID, job.Attempt, w.owner, w.cfg.Lease)
|
||||
cc()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if w.fault(job, "flaky") && job.Attempt <= w.cfg.MaxAttempts {
|
||||
return nil, transientErr("injected_transient", "flaky fault until replay")
|
||||
}
|
||||
if w.fault(job, "transient") && job.Attempt < 3 {
|
||||
return nil, transientErr("injected_transient", "first two attempts fail")
|
||||
}
|
||||
|
||||
data, err := w.readOriginal(pctx, job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
if hex.EncodeToString(sum[:]) != job.SHA256 || int64(len(data)) != job.SizeBytes {
|
||||
return nil, &imaging.PermanentError{Code: "processing_failed", Detail: "staged original checksum mismatch"}
|
||||
}
|
||||
outs, err := imaging.Process(pctx, data, job.ContentType, w.cfg.Limits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := w.s3.Put(pctx, job.ObjectKey, data, job.ContentType); err != nil {
|
||||
return nil, transientErr("storage_unavailable", err.Error())
|
||||
}
|
||||
variants := make([]model.Variant, 0, len(outs))
|
||||
for _, o := range outs {
|
||||
key := fmt.Sprintf("variants/%s/%s/v%d/%s.%s", job.ChatID, job.AttachmentID, job.PipelineVersion, o.Name, o.Ext)
|
||||
if err := w.s3.Put(pctx, key, o.Data, o.ContentType); err != nil {
|
||||
return nil, transientErr("storage_unavailable", err.Error())
|
||||
}
|
||||
vs := sha256.Sum256(o.Data)
|
||||
variants = append(variants, model.Variant{Name: o.Name, ContentType: o.ContentType, SizeBytes: int64(len(o.Data)),
|
||||
Width: o.Width, Height: o.Height, SHA256: hex.EncodeToString(vs[:]), ObjectKey: key})
|
||||
}
|
||||
if w.fault(job, "worker-after-effect") && !redelivered {
|
||||
log.Warn("FAULT: exit after S3 writes before COMMIT")
|
||||
os.Exit(42)
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
func (w *Worker) readOriginal(ctx context.Context, job *store.JobWork) ([]byte, error) {
|
||||
if job.StagingKey != "" {
|
||||
data, err := w.s3.ReadAll(ctx, job.StagingKey, w.cfg.Limits.MaxBytes)
|
||||
if err == nil {
|
||||
return data, nil
|
||||
}
|
||||
if !objstore.IsNotFound(err) {
|
||||
return nil, transientErr("storage_unavailable", err.Error())
|
||||
}
|
||||
}
|
||||
data, err := w.s3.ReadAll(ctx, job.ObjectKey, w.cfg.Limits.MaxBytes)
|
||||
if objstore.IsNotFound(err) {
|
||||
return nil, &imaging.PermanentError{Code: "processing_failed", Detail: "original is missing"}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, transientErr("storage_unavailable", err.Error())
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (w *Worker) dropStaging(job *store.JobWork, log *slog.Logger) {
|
||||
if job.StagingKey == "" {
|
||||
return
|
||||
}
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := w.s3.Remove(c, job.StagingKey); err != nil {
|
||||
log.Warn("staging cleanup deferred to sweeper", "error", err.Error())
|
||||
return
|
||||
}
|
||||
_ = w.st.ClearStaging(c, job.AttachmentID)
|
||||
}
|
||||
|
||||
func (w *Worker) backoff(attempt int) time.Duration {
|
||||
d := w.cfg.BackoffBase << min(attempt-1, 20)
|
||||
if d > w.cfg.BackoffMax || d <= 0 {
|
||||
d = w.cfg.BackoffMax
|
||||
}
|
||||
return d/2 + time.Duration(rand.Int64N(int64(d/2)+1))
|
||||
}
|
||||
|
||||
type transient struct{ code, detail string }
|
||||
|
||||
func (e *transient) Error() string { return e.code + ": " + e.detail }
|
||||
|
||||
func transientErr(code, detail string) error { return &transient{code, detail} }
|
||||
|
||||
func classify(err error) (string, bool) {
|
||||
var p *imaging.PermanentError
|
||||
var t *transient
|
||||
switch {
|
||||
case errors.As(err, &p):
|
||||
return p.Code, true
|
||||
case errors.As(err, &t):
|
||||
return t.code, false
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return "processing_timeout", false
|
||||
}
|
||||
return "processing_failed", false
|
||||
}
|
||||
|
||||
func safeDetail(err error) string {
|
||||
s := err.Error()
|
||||
if i := strings.Index(s, "http"); i >= 0 {
|
||||
s = s[:i] + "[redacted]"
|
||||
}
|
||||
if len(s) > 300 {
|
||||
s = s[:300]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (w *Worker) sweeper(ctx context.Context) {
|
||||
t := time.NewTicker(w.cfg.SweepEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
c, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
if n, err := w.st.RecoverExpiredLeases(c, 100); err == nil && n > 0 {
|
||||
w.log.Warn("expired leases recovered", "count", n)
|
||||
}
|
||||
refs, err := w.st.StaleStaging(c, w.cfg.PendingGrace, w.cfg.StagingRetain, 100)
|
||||
if err == nil {
|
||||
for _, r := range refs {
|
||||
if err := w.s3.Remove(c, r.StagingKey); err != nil && !objstore.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err := w.st.ForgetStaging(c, r); err == nil {
|
||||
w.log.Info("staging cleaned", "attachment_id", r.AttachmentID, "status", r.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
if n, err := w.st.PurgeOutbox(c, 24*time.Hour); err == nil && n > 0 {
|
||||
w.log.Info("published outbox rows purged", "count", n)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.5 KiB |
@@ -0,0 +1,34 @@
|
||||
"""Synthetic PNG builders for negative/limit tests (stdlib only, nothing stored in Git)."""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
body = kind + data
|
||||
return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff)
|
||||
|
||||
|
||||
def png(width, height, raw=None, level=6, color_type=2):
|
||||
"""RGB (color_type=2) PNG; raw is the filtered scanline stream (default: random noise)."""
|
||||
if raw is None:
|
||||
row = 1 + 3 * width
|
||||
raw = b''.join(b'\x00' + os.urandom(row - 1) for _ in range(height))
|
||||
ihdr = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) +
|
||||
chunk(b'IDAT', zlib.compress(raw, level)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def near_limit_png(limit=10 * 1024 * 1024):
|
||||
"""Valid noise PNG a little below the byte limit (stored deflate blocks)."""
|
||||
width = 1000
|
||||
height = (limit - 64 * 1024) // (1 + 3 * width)
|
||||
data = png(width, height, level=0)
|
||||
assert len(data) <= limit, len(data)
|
||||
return data
|
||||
|
||||
|
||||
def bomb_png(width=20000, height=20000):
|
||||
"""Header declares 400M pixels, the compressed payload is tiny."""
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) +
|
||||
chunk(b'IDAT', zlib.compress(b'\x00' * 4096, 9)) + chunk(b'IEND', b''))
|
||||
Reference in New Issue
Block a user