66 changed files with 7731 additions and 18 deletions
+11 -1
View File
@@ -11,7 +11,7 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
@@ -23,3 +23,13 @@ WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
MAX_ACTIVE_JOBS=500
JOB_LEASE_SECONDS=30
JOB_BACKOFF_BASE=1s
JOB_BACKOFF_MAX=60s
WORKER_DECODE_BUDGET_BYTES=402653184
WORKER_MEMORY=768M
WORKER_CPUS=2.0
WORKER_GOMEMLIMIT=600MiB
PENDING_UPLOAD_GRACE=15m
FAULT_INJECTION=1
+25
View File
@@ -0,0 +1,25 @@
# syntax=docker/dockerfile:1
FROM golang:1.23.4-alpine3.21 AS build
WORKDIR /src
COPY src/go.mod src/go.sum ./
RUN go mod download
COPY src/ ./
ENV CGO_ENABLED=0
RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
FROM alpine:3.21.2 AS runtime
RUN addgroup -S app && adduser -S -G app app
WORKDIR /app
COPY --from=build /out/server /app/server
ARG APP_BUILD=dev
ENV APP_BUILD=$APP_BUILD
LABEL org.opencontainers.image.revision=$APP_BUILD
USER app
EXPOSE 8080
ENV APP_HOST=0.0.0.0 APP_PORT=8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1
ENTRYPOINT ["/app/server"]
+39
View File
@@ -28,3 +28,42 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: secrets up down ps unit worker-stop worker-start ops track1
secrets:
$(PYTHON) scripts/init_secrets.py
up: secrets
docker compose --env-file .env up -d --build --wait
down:
docker compose stop
ps:
docker compose ps -a
unit:
docker run --rm -v "$(CURDIR)/src:/src" -w /src golang:1.23.4-alpine3.21 go test ./...
worker-stop:
docker compose stop worker
worker-start:
docker compose start worker
ops:
$(PYTHON) scripts/lab5ops.py worker-down --output evidence/worker-down.json
$(PYTHON) scripts/lab5ops.py api-crash --output evidence/api-crash.json
$(PYTHON) scripts/lab5ops.py effect-crash --output evidence/effect-crash.json
$(PYTHON) scripts/lab5ops.py ack-crash --output evidence/ack-crash.json
$(PYTHON) scripts/lab5ops.py retry --output evidence/retry.json
$(PYTHON) scripts/lab5ops.py replay --output evidence/replay.json
$(PYTHON) scripts/lab5ops.py invalid --output evidence/invalid.json
$(PYTHON) scripts/lab5ops.py hang --output evidence/hang.json
$(PYTHON) scripts/lab5ops.py dlq --output evidence/dlq.json
$(PYTHON) scripts/lab5ops.py limits --output evidence/limits.json
track1:
$(PYTHON) scripts/track1/run_track1.py matrix --output evidence/track1/matrix.json
$(PYTHON) scripts/track1/run_track1.py backpressure --output evidence/track1/backpressure.json
+173 -17
View File
@@ -1,37 +1,193 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Кораблев Петр, БВТ2304
- Целевая оценка: **5** (уровень 3 + уровень 4 + **трек 1 «Очередь под перегрузкой»**)
- Commit/PR: ветка `solution/BVT2304-korablev`
- Использованные источники, библиотеки и помощь ИИ: Go 1.23.4, pgx/v5 5.7.2, go-redis/v9 9.7.0, minio-go/v7 7.0.82 (из lab4); **новые**: `github.com/rabbitmq/amqp091-go` 1.10.0 (AMQP-клиент), `golang.org/x/image` 0.23.0 (`draw.CatmullRom` для ресайза), `github.com/gen2brain/webp` 0.5.2 (libwebp, скомпилированный в WebAssembly и исполняемый через wazero 1.8.1 — кодирование WebP без cgo). Образы: PostgreSQL 17.2-alpine3.21, Valkey 8.0.2-alpine3.21, HAProxy 3.0.5-alpine, SeaweedFS 3.80, **RabbitMQ 4.0.5-management-alpine**, golang:1.23.4-alpine3.21 → alpine:3.21.2, для служебных проверок python:3.12.8-alpine3.21. Документация: RabbitMQ Reliability / Confirms / Quorum queues, EXIF 2.32 (тег Orientation 0x0112). Помощь ИИ использовалась при написании отчёта.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker 29.x + Compose v2 (Docker Desktop/WSL2), Python 3.9+, Git. Go и make локально не нужны (сборка в Docker; без make команды ниже выполняются через `python`). На Windows тесты запускать с `PYTHONUTF8=1`.
```sh
python3 scripts/init_secrets.py # = make secrets: secrets/* + .env из .env.example
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait # = make up; ждёт healthy и завершения migrate/s3-init
make check && make test # smoke через LB http://localhost:8080
make unit # Go unit-тесты imaging (размеры, EXIF, alpha, лимиты)
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
docker compose up -d --no-recreate --wait api # вернуть остановленную реплику
make ops # сценарии отказов уровней 3/4 -> evidence/*.json
make track1 # трек 1 -> evidence/track1/*.json (~15 мин)
```
- `init_secrets.py` дополнительно к lab4 создаёт `secrets/rabbitmq_auth.conf` (пользователь `messenger` со случайным паролем; монтируется как Compose secret в `/etc/rabbitmq/conf.d/20-auth.conf`, гостевой пользователь не создаётся) и `secrets/broker_url` (AMQP URL для API/worker через `BROKER_URL_FILE`). Всё в `secrets/` (в `.gitignore`/`.dockerignore`).
- Порядок старта (`depends_on`): postgres/valkey/s3/rabbitmq healthy → `migrate` (миграции под advisory lock, теперь 0001–0004) → `s3-init` → `api` ×2 и `worker` → `lb`.
- `FAULT_INJECTION=1` в `.env` включает учебные триггеры сбоев по имени файла `fault-*` (см. ниже). Вне стенда выставить `0`.
- Остановка worker для демонстрации: `scripts/stop-worker.sh` / `.cmd` (= `docker compose stop worker`), запуск — `scripts/start-worker.sh` / `.cmd`.
- Безопасная остановка: `docker compose stop` (`make down`), volumes `pgdata`, `s3data`, `mqdata` сохраняются. Намеренное удаление учебных данных: `docker compose down -v` (необратимо).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
client → HAProxy (lb, 127.0.0.1:8080) → api ×2 ──┬─ PostgreSQL: attachments, attachment_jobs, job_outbox,
│ job_events, attachment_variants (+ всё из lab4)
├─ S3 staging/<att_id> (оригинал до обработки)
└─ outbox relay ──publish+confirm──► RabbitMQ
exchange attachments
quorum queue attachment.process.v1
DLX → attachment.process.dlq
worker (отдельный процесс/контейнер) ◄──prefetch=N, manual ack──┘
├─ claim/lease в PostgreSQL → читает staging → пишет в S3 attachments/<chat>/<att> (оригинал)
│ и variants/<chat>/<att>/v1/{thumbnail.jpg,crop.jpg,optimized.webp} → COMMIT результата → удаляет staging → ack
├─ outbox relay (тот же код, что в API; SKIP LOCKED — несколько relay безопасны)
└─ sweeper: истёкшие lease → queued; retention staging; зависшие pending-загрузки
```
**Источник истины** — PostgreSQL: строка `attachment_jobs` определяет, нужно ли что-то делать; сообщение брокера — только «разбудить worker» с `job_id`/`attempt` (`contracts/jobs.schema.json`, без байтов, ключей и URL). Worker берёт доверенные метаданные (ключи, тип, sha256) из БД.
**Upload, транзакционные границы** (`src/internal/api/attachments.go`, `src/internal/store/jobs.go`):
1. аутентификация → членство (чужой — 404) → filename → Content-Type → `Content-Length` ≤ 10 MiB → дешёвая предпроверка размера очереди (`MAX_ACTIVE_JOBS`, иначе 503 + `Retry-After: 5`) → тело через `MaxBytesReader` → magic bytes + `image.DecodeConfig` (только заголовок: >40 Мпикс → 413). **Полное декодирование и ресайз API больше не делает** — только worker, даже если worker недоступен.
2. `INSERT attachments status='pending'` (невидима в API) → `PUT staging/<att_id>` (приватный prefix того же bucket, persistent volume S3).
3. **Одна транзакция**: `pg_advisory_xact_lock` + повторный подсчёт активных jobs (строгая граница), `pending→queued`, `INSERT attachment_jobs`, `INSERT job_outbox`, `INSERT job_events('queued')` → COMMIT → `202 queued`.
Значит, в момент 202 устойчиво сохранены: полный проверенный оригинал в S3 staging, метаданные, job и намерение публикации (outbox). Сообщения в брокере в этот момент может ещё не быть — это нормально.
4. Ошибка на 2/3 → 503 и компенсация (удаляем pending-строку, затем staging-объект). Оставшееся (БД недоступна) убирает sweeper через `PENDING_UPLOAD_GRACE=15m`.
**Transactional outbox** (`src/internal/relay`): relay берёт `job_outbox WHERE published_at IS NULL AND available_at<=now() FOR UPDATE SKIP LOCKED`, публикует persistent-сообщение и **ждёт publisher confirm**, только потом ставит `published_at` в той же транзакции. Relay работает в каждом API (сразу после COMMIT его «пинают») и в worker, плюс опрос раз в секунду. Падение между COMMIT и publish → строку опубликует любой живой relay. Падение после publish до отметки → повторная публикация, т.е. дубль сообщения, который worker обезвреживает (ниже). Недоступность RabbitMQ не ломает upload: 202 продолжает отдаваться, строки копятся в outbox.
**Broker** — RabbitMQ 4.0.5, quorum-очередь (реплицируемый журнал на volume `mqdata`, durable), `x-delivery-limit=20` и DLX → `attachment.process.dlq` для poison/malformed сообщений. Publisher confirms на стороне relay, manual ack на стороне worker, `prefetch = WORKER_CONCURRENCY`.
**Worker** (`src/internal/worker/worker.go`), на каждую доставку:
1. Проверка схемы сообщения; некорректное → `nack(requeue=false)` → DLQ.
2. `ClaimJob(job_id, attempt)` под `SELECT … FOR UPDATE`: `ready/failed` → no-op ack; `attempt` в сообщении ≠ текущему → устаревший дубль, ack; `processing` с живым lease → ack (работу держит другой); retry ещё не наступил → ack. Иначе `processing`, `lease_owner`, `lease_until=now()+JOB_LEASE_SECONDS`. **Redelivery** (флаг `redelivered` от RabbitMQ) разрешает перехват lease немедленно: брокер переотдаёт сообщение только после закрытия канала прежнего держателя (crash/kill), поэтому ждать истечения lease не нужно.
3. Чтение оригинала из staging (если staging уже удалён — из конечного ключа), сверка sha256/size с БД, обработка с ограничениями (ниже), запись оригинала и вариантов **по детерминированным ключам** (`(attachment_id, pipeline_version, variant)` → один ключ S3) — повтор перезаписывает те же байты, второго набора не появляется.
4. `CompleteJob` — одна транзакция: `UPDATE job SET ready WHERE job_id AND attempt AND status='processing'`, `INSERT attachment_variants … ON CONFLICT DO NOTHING`, `attachment → ready`. Устаревшая попытка получает 0 строк и ничего не меняет: ready никогда не откатывается.
5. Удаление staging (после COMMIT), `staging_key=NULL` → **ack**.
**Ошибки и повторы**: `imaging.PermanentError` (`invalid_image`, `image_too_large`, checksum mismatch) → сразу `failed`. Временные (`storage_unavailable`, `processing_timeout`, `lease_expired`, `injected_transient`) → `queued`, `attempt+1`, новая outbox-строка с `available_at = now() + backoff`, backoff экспоненциальный с jitter «equal jitter»: `d = JOB_BACKOFF_BASE·2^(attempt-1)` (cap `JOB_BACKOFF_MAX=60s`), ожидание равномерно в `[d/2, d]`. После `JOB_MAX_ATTEMPTS=5` → `failed / retry_exhausted`. Отложенный повтор реализован через outbox, поэтому не нужен delayed-плагин RabbitMQ и повтор переживает рестарт брокера. Если сама БД недоступна при фиксации — `nack(requeue=true)`.
**Реестр failed и replay**: `attachment_jobs WHERE status='failed'` + аудит `job_events` (каждая попытка, причина, кто сделал replay). Явный операторский replay: `docker compose exec worker /app/server replay <attachment_id> --actor <name>` — только `failed → queued`, тот же `job_id` (тот же логический эффект), новый бюджет попыток, запись в аудит. Для `ready` replay отказывает (`noop`), для `queued/processing` — тоже; если staging уже удалён retention-ом — `refused`. DLQ брокера содержит только сообщения, которые нельзя разобрать (по ним нет job).
**Lease / зависший vs долгий worker**: живой worker продлевает lease (heartbeat каждые `lease/3`), но не дольше `JOB_TIMEOUT_SECONDS=60` — потом сам прерывает попытку как `processing_timeout`. Зависший (нет heartbeat) теряет lease через `JOB_LEASE_SECONDS=30`, sweeper (каждые 5 с) переводит job в `queued` с `attempt+1`. Если «оживший» старый worker всё же досчитает, его `CompleteJob` с прежним `attempt` ничего не изменит. Два одновременных исполнителя возможны только в окне сетевого разрыва; они пишут одинаковые байты по одинаковым ключам, а результат фиксирует ровно один.
**Варианты (pipeline_version=1)**, `src/internal/imaging/imaging.go`: EXIF Orientation (1–8) применяется до ресайза, alpha компонуется на белый, метаданные не копируются ни в один вариант (энкодерам передаются только пиксели).
- `thumbnail`: JPEG q=85, вписан в 256×256 без увеличения, размеры округляются вниз целочисленно (320×200 → 256×160), минимум 1 px.
- `crop`: JPEG q=85, центральный квадрат исходника → 128×128 (маленькие увеличиваются).
- `optimized`: WebP lossy q=80, method=4, вписан в 1280×1280 без увеличения, без EXIF/ICC/XMP. Уменьшение размера маленьких файлов не обещается.
- Ресайз — Catmull-Rom (`x/image/draw`).
**Лимиты worker**: ≤10 MiB bytes (повторная проверка), ≤40 Мпикс по заголовку до декодирования, общий бюджет памяти декодирования `WORKER_DECODE_BUDGET_BYTES=384 MiB` (взвешенный семафор по оценке `пиксели·8 + 6.5 MiB`; таймаутнувшая попытка держит свою долю до реального завершения декодера), время `JOB_TIMEOUT_SECONDS`, контейнер `768M`/`2 CPU`, `GOMEMLIMIT=600MiB`, `prefetch=concurrency`.
**Staging cleanup**: удаляется сразу после COMMIT `ready`; у `failed` хранится `STAGING_RETENTION_SECONDS=86400` (для replay), потом sweeper удаляет; брошенные `pending` — через 15 мин. Текущую работу (`queued/processing`) sweeper не выбирает.
**API-переход 4 → 5**: upload `201 ready` → `202 queued`; `GET /attachments/{id}` показывает `queued/processing/ready/failed`, `variants` только у ready, `error_code` только у failed; `content` (любой variant) для не-ready → 409, отсутствующий variant у ready → 404, неизвестный → 400; к сообщению можно прикрепить queued/processing/ready, failed → 409.
**Что перенесено из lab4 и что изменено.** Перенесены без изменений пользователи, сессии (Argon2id, Valkey-кэш), чаты, сообщения, идемпотентность, keyset-пагинация, `X-Instance-Id`, HAProxy, S3 (SeaweedFS), privacy bucket, хуки restart/stop-one. Изменено: upload (staging + outbox вместо прямого PUT в конечный ключ), из API убрано полное декодирование, `restart.*` теперь перезапускает и RabbitMQ/worker. **Удалён reconciler lab4** (трек 2 lab4): он считал «осиротевшими» все объекты вне `attachments/`, т.е. удалил бы staging и варианты; его функции по staging/pending взял sweeper worker. Сверка S3↔БД для вариантов в этой работе не реализована (см. ограничения).
**Соответствие `.env.example`**: `BROKER_URL_FILE`, `WORKER_CONCURRENCY` (= prefetch), `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS`, `S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` используются как есть; добавлены `MAX_ACTIVE_JOBS`, `JOB_LEASE_SECONDS`, `JOB_BACKOFF_BASE/MAX`, `WORKER_DECODE_BUDGET_BYTES`, `WORKER_MEMORY/CPUS/GOMEMLIMIT`, `PENDING_UPLOAD_GRACE`, `FAULT_INJECTION`. `S3_ENDPOINT` в примере исправлен на `http://s3:8333` (SeaweedFS).
| Роль | Service | Образ | Health | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| LB | lb | haproxy:3.0.5-alpine | `:8404/healthz` | — | app; 127.0.0.1:8080 | — |
| API ×2 | api | mtusi-messenger:lab5 | `/health/ready` (только PostgreSQL) | нет (read-only rootfs) | app, data | database_url, cache_url, broker_url, s3_* |
| Worker | worker | mtusi-messenger:lab5 (`worker`) | — (restart unless-stopped) | нет (read-only rootfs) | data | database_url, broker_url, s3_* |
| Broker | rabbitmq | rabbitmq:4.0.5-management-alpine | `check_port_connectivity` | mqdata | data, не публикуется | rabbitmq_auth.conf |
| БД | postgres | postgres:17.2-alpine3.21 | pg_isready | pgdata | data | pg_password |
| Сессии | valkey | valkey/valkey:8.0.2-alpine3.21 | ping | — | data | valkey_auth |
| S3 | s3 | chrislusf/seaweedfs:3.80 | `:8333/healthz` | s3data | data | s3_identities.json |
| Bootstrap | migrate, s3-init | mtusi-messenger:lab5 | one-shot | — | data | database_url / s3_admin_* |
`/health/ready` зависит только от PostgreSQL: недоступность брокера/worker/S3 не снимает API с балансировки (текстовый мессенджер работает, upload при живом S3 продолжает отдавать 202 через outbox).
**Учебные триггеры сбоев** (`FAULT_INJECTION=1`, срабатывают один раз на файл по префиксу имени): `fault-api-after-commit` (API `os.Exit` после COMMIT до publish/ответа), `fault-worker-after-effect` (после записи S3 до COMMIT), `fault-worker-before-ack` (после COMMIT до ack), `fault-transient` (попытки 1–2 падают), `fault-flaky` (падают все попытки до replay), `fault-hang` (попытка 1 «замерзает» на 3×lease без heartbeat).
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Окружение: Windows 11, Docker Desktop 29.7.2 (WSL2); лимиты: API 1 CPU/512M ×2, worker 2 CPU/768M (`GOMEMLIMIT=600MiB`), RabbitMQ 1 CPU/512M, S3 1 CPU/1G, PostgreSQL 2 CPU/1G. Дата прогона — 2026-11-11/12.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Уровень 3
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke (202 → ready, оригинал, JPEG thumbnail 256×160) | `make test` | `PASS: lab5 public API scenario` | `evidence/smoke.txt` |
| Минимум lab4 сохранён, сохранность после рестарта всех хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` (stop api/worker/lb, restart postgres/valkey/s3/rabbitmq) | `PASS: persistence preserved confirmed data and existing session` | `evidence/persistence.txt` |
| Отказ одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | 117 записей, 0 ошибок, восстановление 3.97 с, `PASS` | `evidence/failover.txt` |
| 202 только после устойчивого приёма; остановка API после ответа не теряет работу | `lab5ops.py worker-down`: worker остановлен → upload → `restart api` + `restart rabbitmq` → `start worker` | `202 queued`; сообщение с queued-вложением 201; content 409; текстовые сообщения 201; в очереди 1 сообщение до и **после** рестарта API+broker; после старта worker ready за 0.34 с | `evidence/worker-down.json` |
| Отдельные broker и worker, durable + at-least-once, ack после эффекта | `compose.yaml` (rabbitmq quorum, worker), сценарии `effect-crash`/`ack-crash` | Сообщение пережило рестарт брокера с volume; повтор доставки после убийства worker завершил ту же job | `evidence/worker-down.json`, `evidence/effect-crash.json` |
| Недоступность worker не вызывает синхронный resize | API не декодирует изображения вообще (`inspectImage` читает только заголовок) | upload при остановленном worker 202, статус `queued` до запуска worker | `evidence/worker-down.json` |
| Состояния queued/processing/ready/failed; thumbnail по контракту; original неизменен | `worker-down`, `invalid`, `replay` | seen `processing→ready`, `failed (invalid_image / retry_exhausted)`; thumbnail реальный JPEG 256×160; sha256 оригинала совпадает | `evidence/*.json` (`ready_check`) |
| Ограниченный retry + финальный failed с безопасным кодом | `lab5ops.py replay` (`JOB_MAX_ATTEMPTS=5`) | 5 попыток → `failed`, `error_code=retry_exhausted`, без stack trace | `evidence/replay.json` |
| Повтор одной job не создаёт второго набора | `effect-crash`, `ack-crash`, `hang` | везде `variant_rows=3`, в S3 `variants/<chat>/<att>/` ровно 3 объекта | `evidence/effect-crash.json`, `ack-crash.json`, `hang.json` |
### Уровень 4
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| crop 128×128 и optimized WebP; параметры описаны | любой ready (`worker-down`), `make unit` | `thumbnail image/jpeg 256x160`, `crop image/jpeg 128x128`, `optimized image/webp 320x200`; unit-тесты: Fit, EXIF orientation 6 (320×200 → thumbnail 160×256), alpha → белый | `evidence/worker-down.json`, `src/internal/imaging/imaging_test.go` |
| Невалидные/огромные изображения ограничены по bytes, пикселям, времени, памяти | `lab5ops.py invalid`, `lab5ops.py limits` | битый PNG с верным заголовком: 202 → `failed invalid_image` за **1** попытку (постоянная ошибка не ретраится); API: pixel bomb 20000×20000 → 413, не-изображение → 415, >10 MiB → 413, `../` в имени → 400; 6000×6000 (36 Мпикс) → ready за 5.2 с, пик памяти worker 491 MiB при лимите 768 MiB | `evidence/invalid.json`, `evidence/limits.json` |
| Разрыв PostgreSQL ↔ broker закрыт outbox; crash между COMMIT и publish | `lab5ops.py api-crash` (`fault-api-after-commit`) | клиент получил 502 (API-процесс умер после COMMIT), API-контейнер перезапущен (`RestartCount 0→1`), строку outbox опубликовал другой relay, job `ready` без ручных действий; события `queued → started → ready` | `evidence/api-crash.json` |
| Backoff с jitter, постоянные vs временные, DLQ/реестр failed, безопасный replay | `lab5ops.py retry`, `replay`, `dlq` | паузы перед попытками 2 и 3: 0.94 с и 1.81 с (диапазоны [0.5,1] и [1,2]); `fault-flaky`: 5 попыток → `failed retry_exhausted`; content 409, прикрепление к сообщению 409; `replay` → `{"result":"replayed","attempt":6}` → ready; повторный replay → `noop: job already ready`; malformed сообщение → `attachment.process.dlq` 0→1 | `evidence/retry.json`, `replay.json`, `dlq.json` |
| Остановка worker после записи объекта, до ack — повтор завершает тот же результат | `lab5ops.py effect-crash` (после S3 до COMMIT), `ack-crash` (после COMMIT до ack) | effect-crash: события `started → taken_over → ready`, worker `RestartCount 0→1`, 3 варианта; ack-crash: `ready` один раз, после рестарта redelivery → лог `job delivery is a no-op result=done redelivered=true` | `evidence/effect-crash.json`, `evidence/ack-crash.json` |
| Timeout/lease для зависшего processing | `lab5ops.py hang` | попытка 1 зависла без heartbeat → через 31 с sweeper: `retry_scheduled lease_expired` → попытка 2 → ready; «оживший» старый исполнитель: `stale attempt finished after takeover; result already owned by newer attempt`, вариантов по-прежнему 3 | `evidence/hang.json` |
| Ограниченная конкуренция/prefetch | `WORKER_CONCURRENCY` = число goroutine = `basic.qos prefetch`; бюджет декодирования; `MAX_ACTIVE_JOBS` | см. трек 1: при `MAX_ACTIVE_JOBS=20` принято ровно 20 из 80 | `evidence/track1/*.json` |
| Очистка staging после безопасной фиксации | все сценарии ready | `staging_removed=true`, в S3 `staging/<id>` 0 объектов, конечный оригинал 1; у failed staging сохраняется для replay | `evidence/worker-down.json` (`s3`) |
### Трек 1 — очередь под перегрузкой
**Гипотеза.** Pipeline изображений CPU-bound: увеличение `WORKER_CONCURRENCY` (= prefetch) повышает throughput почти линейно, пока число параллельных задач не упрётся в CPU-лимит worker (2 CPU), а для больших изображений — в бюджет памяти декодирования (384 MiB ≈ 2 задачи по 20 Мпикс). Дальше рост concurrency не даёт throughput, но увеличивает RSS. Без ограничения принятой работы очередь и время до ready растут неограниченно; с `MAX_ACTIVE_JOBS` лишняя работа отклоняется сразу (503 + `Retry-After`), а принятая гарантированно дренируется.
**Методика** (`scripts/track1/run_track1.py`, `make track1`). Для каждой настройки worker пересоздаётся с `WORKER_CONCURRENCY ∈ {1,2,4}` (те же лимиты 2 CPU / 768M / `GOMEMLIMIT=600MiB`, бюджет декодирования 384 MiB), для каждого размера — новый чат и **открытый burst** N загрузок 8 клиентскими потоками без ожидания обработки:
- S = `tests/fixtures/sample.png` 320×200 (112 КБ), N=120;
- M = 2000×1500 PNG с шумовыми строками (1.17 МБ), N=40;
- L = 5000×4000 PNG-градиент (66 КБ на диске, 20 Мпикс после декодирования), N=16.
Метрики: время до ready каждой job = `finished_at − created_at` строки `attachment_jobs` (часы сервера, без клиентского polling); throughput = ready / (последний finished − первый created); queue lag — выборка каждые 0.5 с: активные jobs (queued+processing), `messages_ready` в RabbitMQ (`rabbitmqctl list_queues`), возраст самой старой незавершённой job; RSS — пик `docker stats` контейнера worker. Прогрев: стенд уже работал, по одному прогону на ячейку (ограничение — см. ниже). Raw: `evidence/track1/matrix.json` (с таймлайном lag), `evidence/track1/backpressure.json`.
| Размер | conc/prefetch | принято / ready / failed | throughput, job/s | p50 / p95 до ready, с | max queue (DB / MQ ready) | max возраст старейшей job, с | пик RSS worker, MiB |
| --- | --- | --- | --- | --- | --- | --- | --- |
| S 320×200 | 1 | 120 / 120 / 0 | 9.38 | 6.3 / 11.3 | 51 / 112 | 10.9 | 59 |
| S 320×200 | 2 | 120 / 120 / 0 | 16.25 | 3.6 / 6.2 | 63 / 64 | 6.4 | 23 |
| S 320×200 | 4 | 120 / 120 / 0 | **23.27** | **2.3 / 3.9** | 61 / 32 | 0.5 | 29 |
| M 2000×1500 | 1 | 40 / 40 / 0 | 0.56 | 36.8 / 68.1 | 36 / 38 | 69.3 | 165 |
| M 2000×1500 | 2 | 40 / 40 / 0 | **1.11** | **19.2 / 35.0** | 36 / 34 | 33.8 | 328 |
| M 2000×1500 | 4 | 40 / 40 / 0 | 1.09 | 21.1 / 35.8 | 36 / 36 | 32.1 | 513 |
| L 5000×4000 | 1 | 16 / 16 / 0 | 0.37 | 24.2 / 42.8 | 16 / 14 | 38.3 | 601 |
| L 5000×4000 | 2 | 16 / 16 / 0 | **0.72** | **13.6 / 22.0** | 16 / 14 | 21.5 | 585 |
| L 5000×4000 | 4 | 16 / 16 / 0 | 0.73 | 13.7 / 21.7 | 16 / 12 | 19.6 | 593 |
**Backpressure и восстановление** (`backpressure`): API с `MAX_ACTIVE_JOBS=20`, worker concurrency 1, burst 80 M-изображений за 1.1 с → **принято ровно 20, 60 × 503 `unavailable` с `Retry-After: 5`**, ни одного другого кода; максимум активных jobs по выборкам 18 ≤ 20. Все 20 принятых дошли до ready (0 failed, 0 retry), очередь опустела за 37.5 с (throughput 0.56 job/s — как в матрице для conc=1), и **сразу после перегрузки** новый upload стал ready за 0.34 с — система вернулась к idle-латентности без ручных действий. Первая версия ограничения (только предпроверка до чтения тела) пропустила 25 из 80 при пределе 20 (гонка check-then-insert между параллельными загрузками) — после этого добавлена повторная проверка в транзакции под `pg_advisory_xact_lock`, граница стала строгой.
**Выводы.**
1. Для мелких изображений накладные расходы (БД-транзакции, S3 PUT ×4, ack) доминируют, а не CPU: throughput растёт 9.4 → 16.3 → 23.3 job/s (×2.5 при ×4 concurrency), p95 падает 11.3 → 3.9 с, RSS остаётся < 60 MiB. Здесь concurrency 4 однозначно лучше.
2. Для M и L рост останавливается на concurrency 2 (= 2 CPU лимита): 1→2 даёт ×2 throughput и ×2 меньше p95, 2→4 — 0 % (1.11 → 1.09 и 0.72 → 0.73), зато RSS для M растёт 328 → 513 MiB. Для L concurrency 4 фактически не включается: бюджет декодирования 384 MiB пропускает только 2 задачи по 20 Мпикс одновременно, поэтому RSS держится ~590 MiB (у потолка `GOMEMLIMIT`), а OOM нет ни в одном прогоне. Гипотеза подтверждена: оптимум concurrency ≈ число CPU для тяжёлых изображений, больше — только трата памяти.
3. Компромисс latency/ресурсы: при фиксированном CPU latency в очереди определяется законом Литтла — время до ready ≈ длина очереди / throughput; при неограниченном приёме p95 растёт линейно с burst (M, conc=1: 68 с для 40 файлов). Ограничение принятой работы делает худшую задержку предсказуемой (≈ `MAX_ACTIVE_JOBS` / throughput: 20 / 0.56 ≈ 36 с, наблюдали max 35 с) ценой явного 503 для клиента.
4. Выбранная конфигурация по умолчанию: `WORKER_CONCURRENCY=2` (=CPU), бюджет 384 MiB как предохранитель памяти, `MAX_ACTIVE_JOBS=500` (для демонстрации backpressure — 20).
**Ограничения измерения.** По одному прогону на ячейку (разброс не оценён, повтор — `make track1`); один хост Windows/WSL2, клиент и стенд делят CPU; RSS — выборки `docker stats` раз в ~1–2 с, кратковременные пики могут быть пропущены; L-изображение синтетическое и хорошо сжимается (нагрузка — декодирование/ресайз, а не передача байтов); размер M-файлов искусственный (шум каждые 8 строк). Выводы относятся к этим лимитам и профилю, а не к production.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| worker остановлен, upload, рестарт API и RabbitMQ (volumes сохранены) | 202, задача не теряется | 202 queued; сообщение в quorum-очереди до и после рестарта; ready через 0.34 с после старта worker | потерь нет |
| API умирает между COMMIT и publish | outbox опубликует | клиент 502, job ready через ~0.2 с (relay другой реплики/worker) | потерь нет; клиент не знает id — повтор upload создаст второе вложение (контракт это допускает) |
| worker умирает после записи S3 до COMMIT | redelivery, перехват lease, тот же результат | `taken_over → ready`, 3 варианта | ~1 с (рестарт контейнера), дублей нет |
| worker умирает после COMMIT до ack | redelivery — no-op | `result=done redelivered=true`, ack | дублей нет |
| worker «завис» | lease истекает, новая попытка | ready через 31 с (lease 30 с) | старая попытка не перезаписала результат |
| временные ошибки | backoff с jitter | 0.94 с, 1.81 с | ready на 3-й попытке |
| постоянная ошибка (битый PNG) | failed без ретраев | `invalid_image` за 1 попытку | — |
| malformed сообщение в брокере | DLQ | DLQ 0→1 | — |
| перегрузка (80 загрузок при лимите 20) | 503 + Retry-After, работа ограничена | 20 принято, 60 × 503 `Retry-After: 5`, после дренажа новый файл ready за 0.34 с | принятые 20 все ready |
| рестарт всех хранилищ (`restart.cmd`) | данные, сессия, файл сохранены | `make test-persistence` PASS | — |
| остановка одной API-реплики | запись продолжается | 0 ошибок, 3.97 с до первой записи после hook | — |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host: RabbitMQ — одна нода (quorum-очередь из одного члена), PostgreSQL и S3 — по одному экземпляру. Выдерживаются рестарты процессов с сохранёнными volumes, а не потеря диска/хоста.
- At-least-once + идемпотентный эффект ≠ exactly-once: один и тот же S3-ключ может быть записан несколько раз (одинаковыми байтами), а дубль сообщения — получен; «ровно один раз» гарантируется только для зафиксированного результата в PostgreSQL.
- При crash между COMMIT и ответом клиент не узнаёт `attachment_id`; без расширения контракта (Idempotency-Key для upload) повтор создаст второе вложение.
- Таймаут не прерывает декодер Go насильно: зависшая попытка освобождает свою долю бюджета памяти только по фактическому завершению декодирования; защиту от бесконечного декодера даёт лимит контейнера.
- Сверка S3 ↔ БД для вариантов (reconciler lab4) не перенесена; объекты от устаревших попыток всегда совпадают с зафиксированными по ключу, но «сиротские» объекты удалённых вложений не собираются.
- Fault injection по имени файла — учебный механизм, выключается `FAULT_INJECTION=0`.
- Worker использует ту же S3-идентичность, что и API; следующий шаг — отдельная идентичность с правами только на нужные prefix-ы.
+296
View File
@@ -0,0 +1,296 @@
name: mtusi-lab5
x-api-env: &api-env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_STORE: ${SESSION_STORE:-valkey}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT: ${S3_TIMEOUT:-15s}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8}
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-60s}
GOMEMLIMIT: ${API_GOMEMLIMIT:-400MiB}
BROKER_URL_FILE: /run/secrets/broker_url
MAX_ACTIVE_JOBS: ${MAX_ACTIVE_JOBS:-500}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
FAULT_INJECTION: ${FAULT_INJECTION:-0}
x-worker-env: &worker-env
DATABASE_URL_FILE: /run/secrets/database_url
DB_MAX_CONNS: "${WORKER_DB_MAX_CONNS:-8}"
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_DECODE_BUDGET_BYTES: ${WORKER_DECODE_BUDGET_BYTES:-402653184}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
JOB_BACKOFF_BASE: ${JOB_BACKOFF_BASE:-1s}
JOB_BACKOFF_MAX: ${JOB_BACKOFF_MAX:-60s}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
PENDING_UPLOAD_GRACE: ${PENDING_UPLOAD_GRACE:-15m}
FAULT_INJECTION: ${FAULT_INJECTION:-0}
GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-600MiB}
services:
postgres:
image: postgres:17.2-alpine3.21
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
secrets: [pg_password]
volumes:
- pgdata:/var/lib/postgresql/data
networks: [data]
shm_size: 256mb
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1G}"}
stop_grace_period: 30s
restart: unless-stopped
valkey:
image: valkey/valkey:8.0.2-alpine3.21
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
volumes:
- ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro
secrets: [valkey_auth]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"]
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: unless-stopped
s3:
image: chrislusf/seaweedfs:3.80
command:
- server
- -dir=/data
- -s3
- -s3.port=8333
- -s3.config=/run/secrets/s3_identities
- -master.volumeSizeLimitMB=64
- -volume.max=100
secrets: [s3_identities]
volumes:
- s3data:/data
networks: [data]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 2s
timeout: 3s
start_period: 60s
retries: 60
deploy:
resources:
limits: {cpus: "1.0", memory: "${S3_MEMORY:-1G}"}
stop_grace_period: 20s
restart: unless-stopped
s3-init:
image: mtusi-messenger:lab5
pull_policy: never
command: ["s3-init"]
healthcheck:
disable: true
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets: [s3_admin_access_key_id, s3_admin_secret_access_key]
networks: [data]
depends_on:
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully
deploy:
resources:
limits: {cpus: "0.5", memory: 64M}
restart: "no"
migrate:
build:
context: .
args:
APP_BUILD: ${APP_BUILD:-dev}
image: mtusi-messenger:lab5
command: ["migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: "no"
api:
image: mtusi-messenger:lab5
pull_policy: never
environment: *api-env
read_only: true
tmpfs:
- /tmp:size=1m
secrets: [database_url, cache_url, broker_url, s3_access_key_id, s3_secret_access_key]
networks: [app, data]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
postgres:
condition: service_healthy
valkey:
condition: service_healthy
rabbitmq:
condition: service_started
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
start_period: 5s
retries: 3
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"}
stop_grace_period: 15s
restart: unless-stopped
rabbitmq:
image: rabbitmq:4.0.5-management-alpine
hostname: rabbitmq
secrets:
- source: rabbitmq_auth
target: /etc/rabbitmq/conf.d/20-auth.conf
volumes:
- mqdata:/var/lib/rabbitmq
networks: [data]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
start_period: 20s
retries: 30
deploy:
resources:
limits: {cpus: "1.0", memory: 512M}
stop_grace_period: 30s
restart: unless-stopped
worker:
image: mtusi-messenger:lab5
pull_policy: never
command: ["worker"]
environment: *worker-env
read_only: true
tmpfs:
- /tmp:size=1m
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-768M}"}
stop_grace_period: 30s
restart: unless-stopped
lb:
image: haproxy:3.0.5-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
- "[::1]:${APP_PORT:-8080}:8080"
networks: [app]
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 15
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128M}
restart: unless-stopped
networks:
app: {}
data: {}
volumes:
pgdata: {}
s3data: {}
mqdata: {}
secrets:
pg_password:
file: ./secrets/pg_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
valkey_auth:
file: ./secrets/valkey_auth.conf
s3_identities:
file: ./secrets/s3_identities.json
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
rabbitmq_auth:
file: ./secrets/rabbitmq_auth.conf
broker_url:
file: ./secrets/broker_url
+72
View File
@@ -0,0 +1,72 @@
{
"scenario": "ack-crash",
"result": "PASS",
"started_at": "2026-11-11T23:24:36Z",
"fault": "worker os.Exit after result COMMIT + staging cleanup, before ack",
"attachment_id": "fc38c3d8-b1b7-440f-b6c5-9b315979731b",
"upload": 202,
"worker_restart_counts_before": {
"d3259669fad0": 1
},
"worker_restart_counts_after": {
"d3259669fad0": 2
},
"statuses_seen": [
"processing",
"ready"
],
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:24:38.431"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:24:38.460"
},
{
"attempt": "1",
"event": "ready",
"error_code": "",
"at": "23:24:38.529"
}
],
"job_after": {
"job_id": "4b9eb573-b6b4-4f6c-bec9-0d73147bab15",
"status": "ready",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": true
},
"s3": {
"variants_objects": 3,
"final_original": 1,
"staging_objects": 0
},
"redelivery_noop_log": [
{
"result": "done",
"redelivered": true
}
],
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
}
}
+73
View File
@@ -0,0 +1,73 @@
{
"scenario": "api-crash",
"result": "PASS",
"started_at": "2026-11-11T23:24:10Z",
"fault": "API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response",
"attachment_id": "4e18930b-bd0f-4db5-a033-35fbd74c93a1",
"client_view": "client saw failure: -a15d-9d5fe0759a8c/attachments: expected [202], received 502",
"job_right_after_crash": {
"job_id": "8dadbdde-897c-4acb-ae12-b7565b03dbb4",
"status": "ready",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": true
},
"api_restart_counts_before": {
"78c8984ff002": 0,
"4c38672b42d2": 0
},
"api_restart_counts_after": {
"78c8984ff002": 1,
"4c38672b42d2": 0
},
"statuses_seen": [
"ready"
],
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:24:12.312"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:24:12.463"
},
{
"attempt": "1",
"event": "ready",
"error_code": "",
"at": "23:24:12.538"
}
],
"job_after": {
"job_id": "8dadbdde-897c-4acb-ae12-b7565b03dbb4",
"status": "ready",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": true
},
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
}
}
+21
View File
@@ -0,0 +1,21 @@
{
"scenario": "dlq",
"result": "PASS",
"started_at": "2026-11-11T23:25:29Z",
"published": {
"routed": true
},
"dlq_before": {
"ready": 0,
"unacked": 0,
"consumers": 0
},
"dlq_after": {
"ready": 1,
"unacked": 0,
"consumers": 0
},
"worker_log": [
"malformed job message -> DLQ"
]
}
+74
View File
@@ -0,0 +1,74 @@
{
"scenario": "effect-crash",
"result": "PASS",
"started_at": "2026-11-11T23:24:19Z",
"fault": "worker os.Exit after writing original+variants to S3, before result COMMIT and ack",
"attachment_id": "4e2f8752-7072-4dc0-ae24-eccb7f6d25ff",
"upload": 202,
"worker_restart_counts_before": {
"d3259669fad0": 0
},
"worker_restart_counts_after": {
"d3259669fad0": 1
},
"statuses_seen": [
"queued",
"processing",
"ready"
],
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:24:21.431"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:24:21.482"
},
{
"attempt": "1",
"event": "taken_over",
"error_code": "",
"at": "23:24:22.154"
},
{
"attempt": "1",
"event": "ready",
"error_code": "",
"at": "23:24:22.573"
}
],
"job_after": {
"job_id": "f945f7fb-dd8f-461b-b3e5-7cae6db38fcc",
"status": "ready",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": true
},
"s3": {
"variants_objects": 3,
"final_original": 1,
"staging_objects": 0
},
"redelivery_noop_log": [],
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
}
}
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 117, "failed_attempts": 0, "first_recovery_seconds": 3.972}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+77
View File
@@ -0,0 +1,77 @@
{
"scenario": "hang",
"result": "PASS",
"started_at": "2026-11-11T23:25:57Z",
"fault": "attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)",
"attachment_id": "9eb5e502-1491-4ae3-93ae-5031997a921d",
"seconds_to_ready": 31.6,
"statuses_seen": [
"queued",
"processing",
"queued",
"ready"
],
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:25:57.874"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:25:57.929"
},
{
"attempt": "1",
"event": "retry_scheduled",
"error_code": "lease_expired",
"at": "23:26:29.068"
},
{
"attempt": "2",
"event": "started",
"error_code": "",
"at": "23:26:29.319"
},
{
"attempt": "2",
"event": "ready",
"error_code": "",
"at": "23:26:29.391"
}
],
"job_after": {
"job_id": "f955fe0d-3b17-42bf-8df8-aae785e1ae68",
"status": "ready",
"attempt": 2,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 2,
"outbox_unpublished": 0,
"staging_removed": true
},
"s3": {
"variants_objects": 3,
"final_original": 1,
"staging_objects": 0
},
"stale_attempt_logs": [
"stale attempt finished after takeover; result already owned by newer attempt"
],
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
}
}
+50
View File
@@ -0,0 +1,50 @@
{
"scenario": "invalid",
"result": "PASS",
"started_at": "2026-11-11T23:25:26Z",
"corrupt_png": {
"upload": 202,
"statuses_seen": [
"processing",
"failed"
],
"error_code": "invalid_image",
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:25:26.980"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:25:27.016"
},
{
"attempt": "1",
"event": "failed",
"error_code": "invalid_image",
"at": "23:25:27.027"
}
],
"job": {
"job_id": "7ead7c8c-0dd7-4b4a-8c2a-807323b1673a",
"status": "failed",
"attempt": 1,
"max_attempts": 5,
"error_code": "invalid_image",
"variant_rows": 0,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": false
}
},
"api_rejects_before_queue": {
"pixel_bomb_20000x20000": 413,
"not_an_image": 415,
"over_10MiB": 413,
"path_in_filename": 400
}
}
+29
View File
@@ -0,0 +1,29 @@
{
"scenario": "limits",
"result": "PASS",
"started_at": "2026-11-11T23:25:46Z",
"image": "6000x6000 PNG (36 MPx, 116950 bytes)",
"seconds_to_ready": 5.2,
"worker_peak_mem_mib": 490.6,
"worker_mem_limit_mib": 768,
"variants": {
"thumbnail": "256x256",
"crop": "128x128",
"optimized": "1280x1280"
},
"statuses_seen": [
"processing",
"ready"
],
"job": {
"job_id": "a0763969-5958-4662-b71a-5e861560a79b",
"status": "ready",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": true
}
}
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+154
View File
@@ -0,0 +1,154 @@
{
"scenario": "replay",
"result": "PASS",
"started_at": "2026-11-11T23:28:58Z",
"fault": "transient error on every attempt <= JOB_MAX_ATTEMPTS",
"attachment_id": "adcb5aa2-0565-4bc7-b88c-c44cab442533",
"statuses_seen": [
"queued",
"processing",
"queued",
"failed"
],
"failed_metadata": {
"status": "failed",
"error_code": "retry_exhausted",
"variants": []
},
"content_when_failed": 409,
"attach_failed_to_message": 409,
"job_failed": {
"job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d",
"status": "failed",
"attempt": 5,
"max_attempts": 5,
"error_code": "retry_exhausted",
"variant_rows": 0,
"outbox_rows": 5,
"outbox_unpublished": 0,
"staging_removed": false
},
"dlq_registry_query": "1",
"replay_1": {
"attempt": 6,
"job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d",
"result": "replayed"
},
"replay_2_after_ready": {
"result": "noop",
"reason": "job already ready; replay refused"
},
"statuses_after_replay": [
"processing",
"ready"
],
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:28:59.257"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:28:59.318"
},
{
"attempt": "1",
"event": "retry_scheduled",
"error_code": "injected_transient",
"at": "23:28:59.324"
},
{
"attempt": "2",
"event": "started",
"error_code": "",
"at": "23:29:00.268"
},
{
"attempt": "2",
"event": "retry_scheduled",
"error_code": "injected_transient",
"at": "23:29:00.277"
},
{
"attempt": "3",
"event": "started",
"error_code": "",
"at": "23:29:02.077"
},
{
"attempt": "3",
"event": "retry_scheduled",
"error_code": "injected_transient",
"at": "23:29:02.085"
},
{
"attempt": "4",
"event": "started",
"error_code": "",
"at": "23:29:05.701"
},
{
"attempt": "4",
"event": "retry_scheduled",
"error_code": "injected_transient",
"at": "23:29:05.708"
},
{
"attempt": "5",
"event": "started",
"error_code": "",
"at": "23:29:11.048"
},
{
"attempt": "5",
"event": "failed",
"error_code": "retry_exhausted",
"at": "23:29:11.060"
},
{
"attempt": "6",
"event": "replay",
"error_code": "",
"at": "23:29:13.048"
},
{
"attempt": "6",
"event": "started",
"error_code": "",
"at": "23:29:13.264"
},
{
"attempt": "6",
"event": "ready",
"error_code": "",
"at": "23:29:13.784"
}
],
"job_after": {
"job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d",
"status": "ready",
"attempt": 6,
"max_attempts": 10,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 6,
"outbox_unpublished": 0,
"staging_removed": true
},
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
}
}
+87
View File
@@ -0,0 +1,87 @@
{
"scenario": "retry",
"result": "PASS",
"started_at": "2026-11-11T23:24:52Z",
"fault": "attempts 1 and 2 raise a transient error (injected_transient)",
"attachment_id": "1eadb246-c3b9-4152-b21e-1b5ec2de9033",
"statuses_seen": [
"queued",
"processing",
"ready"
],
"events": [
{
"attempt": "1",
"event": "queued",
"error_code": "",
"at": "23:24:53.308"
},
{
"attempt": "1",
"event": "started",
"error_code": "",
"at": "23:24:53.381"
},
{
"attempt": "1",
"event": "retry_scheduled",
"error_code": "injected_transient",
"at": "23:24:53.390"
},
{
"attempt": "2",
"event": "started",
"error_code": "",
"at": "23:24:54.330"
},
{
"attempt": "2",
"event": "retry_scheduled",
"error_code": "injected_transient",
"at": "23:24:54.339"
},
{
"attempt": "3",
"event": "started",
"error_code": "",
"at": "23:24:56.146"
},
{
"attempt": "3",
"event": "ready",
"error_code": "",
"at": "23:24:56.565"
}
],
"retry_gaps_seconds": [
0.94,
1.81
],
"expected_gap_ranges": [
"attempt1: [0.5, 1.0] s",
"attempt2: [1.0, 2.0] s"
],
"job_after": {
"job_id": "09973bd1-2b97-43c2-9292-d3519960a2c7",
"status": "ready",
"attempt": 3,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 3,
"outbox_unpublished": 0,
"staging_removed": true
},
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
}
}
+1
View File
@@ -0,0 +1 @@
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+106
View File
@@ -0,0 +1,106 @@
{
"experiment": "backpressure",
"started_at": "2026-11-12T07:56:12Z",
"worker_limits": "cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB",
"host": "Windows 11, Docker Desktop 29.7.2 (WSL2)",
"max_active_jobs_bound": 20,
"worker_concurrency": 1,
"uploads": 80,
"accepted_202": 20,
"rejected_503": 60,
"retry_after_header": [
"5"
],
"other": [],
"upload_burst_s": 1.22,
"drained": true,
"drain_s": 36.4,
"max_active_observed": 18,
"jobs": {
"jobs": 21,
"ready": 21,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 0.58,
"pipeline_span_s": 36.38,
"time_to_ready_s": {
"p50": 17.728,
"p95": 32.937,
"max": 34.657
}
},
"after_overload_upload_status": "ready",
"after_overload_seconds_to_ready": 0.28,
"lag_timeline": [
{
"t": 3.8,
"active": 8,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 0.29
},
{
"t": 7.7,
"active": 18,
"mq_ready": 18,
"mq_unacked": 1,
"oldest_age_s": 4.38
},
{
"t": 12.1,
"active": 16,
"mq_ready": 15,
"mq_unacked": 1,
"oldest_age_s": 8.2
},
{
"t": 16.0,
"active": 14,
"mq_ready": 12,
"mq_unacked": 1,
"oldest_age_s": 12.58
},
{
"t": 20.2,
"active": 11,
"mq_ready": 9,
"mq_unacked": 1,
"oldest_age_s": 16.32
},
{
"t": 23.8,
"active": 9,
"mq_ready": 9,
"mq_unacked": 1,
"oldest_age_s": 20.4
},
{
"t": 28.3,
"active": 7,
"mq_ready": 6,
"mq_unacked": 1,
"oldest_age_s": 24.09
},
{
"t": 31.5,
"active": 4,
"mq_ready": 3,
"mq_unacked": 1,
"oldest_age_s": 28.48
},
{
"t": 36.5,
"active": 2,
"mq_ready": 0,
"mq_unacked": 1,
"oldest_age_s": 31.68
},
{
"t": 40.0,
"active": 0,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 0.0
}
]
}
+682
View File
@@ -0,0 +1,682 @@
{
"experiment": "matrix",
"started_at": "2026-11-11T23:38:36Z",
"worker_limits": "cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB",
"host": "Windows 11, Docker Desktop 29.7.2 (WSL2)",
"runs": [
{
"size": "S_320x200",
"image_bytes": 111817,
"worker_concurrency_prefetch": 1,
"uploads": 120,
"accepted_202": 120,
"rejected": 0,
"upload_burst_s": 1.09,
"drained": true,
"jobs": 120,
"ready": 120,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 9.38,
"pipeline_span_s": 12.79,
"time_to_ready_s": {
"p50": 6.325,
"p95": 11.297,
"max": 11.759
},
"queue_lag": {
"max_active_jobs": 51,
"max_mq_ready": 112,
"max_oldest_job_age_s": 10.94
},
"worker_peak_rss_mib": 59.3,
"lag_timeline": [
{
"t": 6.9,
"active": 50,
"mq_ready": 112,
"mq_unacked": 1,
"oldest_age_s": 0.51
},
{
"t": 11.0,
"active": 51,
"mq_ready": 64,
"mq_unacked": 1,
"oldest_age_s": 7.09
},
{
"t": 16.8,
"active": 10,
"mq_ready": 12,
"mq_unacked": 1,
"oldest_age_s": 10.94
}
]
},
{
"size": "M_2000x1500",
"image_bytes": 1165217,
"worker_concurrency_prefetch": 1,
"uploads": 40,
"accepted_202": 40,
"rejected": 0,
"upload_burst_s": 1.23,
"drained": true,
"jobs": 40,
"ready": 40,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 0.56,
"pipeline_span_s": 70.84,
"time_to_ready_s": {
"p50": 36.797,
"p95": 68.11,
"max": 69.761
},
"queue_lag": {
"max_active_jobs": 36,
"max_mq_ready": 38,
"max_oldest_job_age_s": 69.31
},
"worker_peak_rss_mib": 165.3,
"lag_timeline": [
{
"t": 6.6,
"active": 14,
"mq_ready": 38,
"mq_unacked": 1,
"oldest_age_s": 0.39
},
{
"t": 10.5,
"active": 36,
"mq_ready": 36,
"mq_unacked": 1,
"oldest_age_s": 7.22
},
{
"t": 16.4,
"active": 34,
"mq_ready": 33,
"mq_unacked": 1,
"oldest_age_s": 11.15
},
{
"t": 20.7,
"active": 31,
"mq_ready": 30,
"mq_unacked": 1,
"oldest_age_s": 16.91
},
{
"t": 25.2,
"active": 28,
"mq_ready": 27,
"mq_unacked": 1,
"oldest_age_s": 21.14
},
{
"t": 30.4,
"active": 26,
"mq_ready": 24,
"mq_unacked": 1,
"oldest_age_s": 25.53
},
{
"t": 34.5,
"active": 23,
"mq_ready": 21,
"mq_unacked": 1,
"oldest_age_s": 30.62
},
{
"t": 40.8,
"active": 21,
"mq_ready": 19,
"mq_unacked": 1,
"oldest_age_s": 34.71
},
{
"t": 44.9,
"active": 17,
"mq_ready": 16,
"mq_unacked": 1,
"oldest_age_s": 40.9
},
{
"t": 51.7,
"active": 15,
"mq_ready": 13,
"mq_unacked": 1,
"oldest_age_s": 44.93
},
{
"t": 55.8,
"active": 11,
"mq_ready": 10,
"mq_unacked": 1,
"oldest_age_s": 51.73
},
{
"t": 61.0,
"active": 9,
"mq_ready": 7,
"mq_unacked": 1,
"oldest_age_s": 55.64
},
{
"t": 65.5,
"active": 6,
"mq_ready": 5,
"mq_unacked": 1,
"oldest_age_s": 60.82
},
{
"t": 69.7,
"active": 3,
"mq_ready": 2,
"mq_unacked": 1,
"oldest_age_s": 65.23
},
{
"t": 75.2,
"active": 1,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 69.31
}
]
},
{
"size": "L_5000x4000",
"image_bytes": 66065,
"worker_concurrency_prefetch": 1,
"uploads": 16,
"accepted_202": 16,
"rejected": 0,
"upload_burst_s": 0.18,
"drained": true,
"jobs": 16,
"ready": 16,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 0.37,
"pipeline_span_s": 42.92,
"time_to_ready_s": {
"p50": 24.227,
"p95": 42.775,
"max": 42.775
},
"queue_lag": {
"max_active_jobs": 16,
"max_mq_ready": 14,
"max_oldest_job_age_s": 38.34
},
"worker_peak_rss_mib": 601.1,
"lag_timeline": [
{
"t": 3.4,
"active": 16,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 0.4
},
{
"t": 8.4,
"active": 15,
"mq_ready": 14,
"mq_unacked": 1,
"oldest_age_s": 4.38
},
{
"t": 13.3,
"active": 13,
"mq_ready": 12,
"mq_unacked": 1,
"oldest_age_s": 9.25
},
{
"t": 17.9,
"active": 11,
"mq_ready": 10,
"mq_unacked": 1,
"oldest_age_s": 14.16
},
{
"t": 23.4,
"active": 10,
"mq_ready": 8,
"mq_unacked": 1,
"oldest_age_s": 18.7
},
{
"t": 27.3,
"active": 8,
"mq_ready": 6,
"mq_unacked": 1,
"oldest_age_s": 24.23
},
{
"t": 33.6,
"active": 6,
"mq_ready": 5,
"mq_unacked": 1,
"oldest_age_s": 28.08
},
{
"t": 37.6,
"active": 4,
"mq_ready": 3,
"mq_unacked": 1,
"oldest_age_s": 34.53
},
{
"t": 44.6,
"active": 2,
"mq_ready": 1,
"mq_unacked": 1,
"oldest_age_s": 38.34
},
{
"t": 48.5,
"active": 0,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 0.0
}
]
},
{
"size": "S_320x200",
"image_bytes": 111817,
"worker_concurrency_prefetch": 2,
"uploads": 120,
"accepted_202": 120,
"rejected": 0,
"upload_burst_s": 1.05,
"drained": true,
"jobs": 120,
"ready": 120,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 16.25,
"pipeline_span_s": 7.39,
"time_to_ready_s": {
"p50": 3.649,
"p95": 6.158,
"max": 6.383
},
"queue_lag": {
"max_active_jobs": 63,
"max_mq_ready": 64,
"max_oldest_job_age_s": 6.35
},
"worker_peak_rss_mib": 22.8,
"lag_timeline": [
{
"t": 6.5,
"active": 63,
"mq_ready": 64,
"mq_unacked": 2,
"oldest_age_s": 0.51
},
{
"t": 10.6,
"active": 1,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 6.35
}
]
},
{
"size": "M_2000x1500",
"image_bytes": 1165247,
"worker_concurrency_prefetch": 2,
"uploads": 40,
"accepted_202": 40,
"rejected": 0,
"upload_burst_s": 1.18,
"drained": true,
"jobs": 40,
"ready": 40,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 1.11,
"pipeline_span_s": 36.02,
"time_to_ready_s": {
"p50": 19.159,
"p95": 35.018,
"max": 35.103
},
"queue_lag": {
"max_active_jobs": 36,
"max_mq_ready": 34,
"max_oldest_job_age_s": 33.77
},
"worker_peak_rss_mib": 328.2,
"lag_timeline": [
{
"t": 3.6,
"active": 11,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 0.27
},
{
"t": 8.7,
"active": 36,
"mq_ready": 34,
"mq_unacked": 2,
"oldest_age_s": 4.26
},
{
"t": 13.5,
"active": 30,
"mq_ready": 28,
"mq_unacked": 2,
"oldest_age_s": 9.11
},
{
"t": 17.5,
"active": 26,
"mq_ready": 23,
"mq_unacked": 2,
"oldest_age_s": 13.82
},
{
"t": 23.2,
"active": 20,
"mq_ready": 17,
"mq_unacked": 2,
"oldest_age_s": 17.73
},
{
"t": 27.3,
"active": 14,
"mq_ready": 12,
"mq_unacked": 2,
"oldest_age_s": 23.2
},
{
"t": 33.9,
"active": 10,
"mq_ready": 6,
"mq_unacked": 2,
"oldest_age_s": 27.16
},
{
"t": 37.9,
"active": 2,
"mq_ready": 0,
"mq_unacked": 2,
"oldest_age_s": 33.77
},
{
"t": 44.8,
"active": 0,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 0.0
}
]
},
{
"size": "L_5000x4000",
"image_bytes": 66065,
"worker_concurrency_prefetch": 2,
"uploads": 16,
"accepted_202": 16,
"rejected": 0,
"upload_burst_s": 0.18,
"drained": true,
"jobs": 16,
"ready": 16,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 0.72,
"pipeline_span_s": 22.17,
"time_to_ready_s": {
"p50": 13.631,
"p95": 22.04,
"max": 22.04
},
"queue_lag": {
"max_active_jobs": 16,
"max_mq_ready": 14,
"max_oldest_job_age_s": 21.51
},
"worker_peak_rss_mib": 585.2,
"lag_timeline": [
{
"t": 6.3,
"active": 16,
"mq_ready": 14,
"mq_unacked": 2,
"oldest_age_s": 0.41
},
{
"t": 10.4,
"active": 12,
"mq_ready": 10,
"mq_unacked": 2,
"oldest_age_s": 7.17
},
{
"t": 16.7,
"active": 8,
"mq_ready": 6,
"mq_unacked": 2,
"oldest_age_s": 11.19
},
{
"t": 20.7,
"active": 4,
"mq_ready": 2,
"mq_unacked": 2,
"oldest_age_s": 17.55
},
{
"t": 25.8,
"active": 2,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 21.51
}
]
},
{
"size": "S_320x200",
"image_bytes": 111817,
"worker_concurrency_prefetch": 4,
"uploads": 120,
"accepted_202": 120,
"rejected": 0,
"upload_burst_s": 1.11,
"drained": true,
"jobs": 120,
"ready": 120,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 23.27,
"pipeline_span_s": 5.16,
"time_to_ready_s": {
"p50": 2.281,
"p95": 3.945,
"max": 4.099
},
"queue_lag": {
"max_active_jobs": 61,
"max_mq_ready": 32,
"max_oldest_job_age_s": 0.51
},
"worker_peak_rss_mib": 29.0,
"lag_timeline": [
{
"t": 4.9,
"active": 61,
"mq_ready": 32,
"mq_unacked": 4,
"oldest_age_s": 0.51
},
{
"t": 9.0,
"active": 0,
"mq_ready": 32,
"mq_unacked": 4,
"oldest_age_s": 0.0
}
]
},
{
"size": "M_2000x1500",
"image_bytes": 1165242,
"worker_concurrency_prefetch": 4,
"uploads": 40,
"accepted_202": 40,
"rejected": 0,
"upload_burst_s": 1.28,
"drained": true,
"jobs": 40,
"ready": 40,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 1.09,
"pipeline_span_s": 36.86,
"time_to_ready_s": {
"p50": 21.093,
"p95": 35.792,
"max": 35.83
},
"queue_lag": {
"max_active_jobs": 36,
"max_mq_ready": 36,
"max_oldest_job_age_s": 32.06
},
"worker_peak_rss_mib": 513.1,
"lag_timeline": [
{
"t": 5.5,
"active": 9,
"mq_ready": 36,
"mq_unacked": 4,
"oldest_age_s": 0.28
},
{
"t": 9.4,
"active": 36,
"mq_ready": 31,
"mq_unacked": 4,
"oldest_age_s": 6.15
},
{
"t": 13.9,
"active": 32,
"mq_ready": 24,
"mq_unacked": 4,
"oldest_age_s": 9.87
},
{
"t": 19.0,
"active": 27,
"mq_ready": 20,
"mq_unacked": 4,
"oldest_age_s": 14.19
},
{
"t": 22.8,
"active": 20,
"mq_ready": 20,
"mq_unacked": 4,
"oldest_age_s": 19.04
},
{
"t": 28.5,
"active": 16,
"mq_ready": 8,
"mq_unacked": 4,
"oldest_age_s": 22.81
},
{
"t": 32.3,
"active": 12,
"mq_ready": 8,
"mq_unacked": 4,
"oldest_age_s": 28.43
},
{
"t": 39.0,
"active": 6,
"mq_ready": 0,
"mq_unacked": 0,
"oldest_age_s": 32.06
}
]
},
{
"size": "L_5000x4000",
"image_bytes": 66065,
"worker_concurrency_prefetch": 4,
"uploads": 16,
"accepted_202": 16,
"rejected": 0,
"upload_burst_s": 0.18,
"drained": true,
"jobs": 16,
"ready": 16,
"failed": 0,
"retried_jobs": 0,
"throughput_jobs_per_s": 0.73,
"pipeline_span_s": 21.84,
"time_to_ready_s": {
"p50": 13.692,
"p95": 21.692,
"max": 21.692
},
"queue_lag": {
"max_active_jobs": 16,
"max_mq_ready": 12,
"max_oldest_job_age_s": 19.61
},
"worker_peak_rss_mib": 593.2,
"lag_timeline": [
{
"t": 4.9,
"active": 16,
"mq_ready": 12,
"mq_unacked": 4,
"oldest_age_s": 0.42
},
{
"t": 8.9,
"active": 12,
"mq_ready": 8,
"mq_unacked": 4,
"oldest_age_s": 5.73
},
{
"t": 15.0,
"active": 10,
"mq_ready": 4,
"mq_unacked": 4,
"oldest_age_s": 9.71
},
{
"t": 18.9,
"active": 6,
"mq_ready": 2,
"mq_unacked": 4,
"oldest_age_s": 15.7
},
{
"t": 25.9,
"active": 2,
"mq_ready": 0,
"mq_unacked": 2,
"oldest_age_s": 19.61
}
]
}
]
}
+73
View File
@@ -0,0 +1,73 @@
{
"scenario": "worker-down",
"result": "PASS",
"started_at": "2026-11-11T23:23:18Z",
"attachment_id": "4ebdeb92-a3d0-467d-b8cf-e7e7d45306c2",
"message_id": "f0e4acb4-49b4-4c27-b726-911937e01552",
"upload_status": 202,
"upload_ms_worker_down": 99,
"queue_before_restart": {
"ready": 1,
"unacked": 0,
"consumers": 0
},
"job_before_restart": {
"job_id": "a56aa984-d3eb-4716-a47d-f246d752426c",
"status": "queued",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 0,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": false
},
"queue_after_api_and_broker_restart": {
"ready": 1,
"unacked": 0,
"consumers": 0
},
"status_after_restart_before_worker": "queued",
"seconds_to_ready_after_worker_start": 0.34,
"statuses_seen": [
"processing",
"ready"
],
"steps": [
"docker compose stop worker",
"upload -> 202 queued",
"message with queued attachment -> 201",
"content while queued -> 409",
"plain text message -> 201",
"docker compose restart api rabbitmq (volumes kept)",
"docker compose start worker"
],
"ready_check": {
"variants": {
"thumbnail": "image/jpeg 256x160 5620B",
"crop": "image/jpeg 128x128 2440B",
"optimized": "image/webp 320x200 2462B"
},
"thumbnail_real_jpeg_size": [
256,
160
],
"original_sha256_ok": true
},
"job_after": {
"job_id": "a56aa984-d3eb-4716-a47d-f246d752426c",
"status": "ready",
"attempt": 1,
"max_attempts": 5,
"error_code": "",
"variant_rows": 3,
"outbox_rows": 1,
"outbox_unpublished": 0,
"staging_removed": true
},
"s3": {
"variants_objects": 3,
"final_original": 1,
"staging_objects": 0
}
}
+45
View File
@@ -0,0 +1,45 @@
global
log stdout format raw local0 info
maxconn 4096
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold obsolete 5s
accepted_payload_size 8192
defaults
mode http
log global
option httplog
option dontlognull
timeout connect 1s
timeout client 75s
timeout server 75s
timeout http-request 5s
timeout http-keep-alive 10s
timeout check 500ms
retries 2
retry-on conn-failure
option redispatch 1
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 init-addr none resolvers docker resolve-prefer ipv4
server-template api 8 api:8080
frontend internal_stats
bind :8404
monitor-uri /healthz
stats enable
stats uri /stats
stats refresh 5s
+8
View File
@@ -0,0 +1,8 @@
bind 0.0.0.0
port 6379
protected-mode yes
save ""
appendonly no
maxmemory 64mb
maxmemory-policy volatile-ttl
include /run/secrets/valkey_auth
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
import json
import secrets
import shutil
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / 'secrets'
def write(name, value):
path = SECRETS / name
if path.exists():
print('keep secrets/' + name)
return path.read_text().strip()
path.write_bytes((value + '\n').encode('utf-8'))
print('created secrets/' + name)
return value
def main():
SECRETS.mkdir(exist_ok=True)
pg = write('pg_password', secrets.token_urlsafe(24))
write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg)
vk_conf = SECRETS / 'valkey_auth.conf'
if vk_conf.exists():
vk = vk_conf.read_text().split()[-1]
print('keep secrets/valkey_auth.conf')
else:
vk = secrets.token_urlsafe(24)
write('valkey_auth.conf', 'requirepass ' + vk)
write('cache_url', 'redis://:%s@valkey:6379/0' % vk)
admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8))
admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30))
api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8))
api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30))
bucket = 'messenger-attachments'
identities = {'identities': [
{'name': 'bootstrap-admin',
'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}],
'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']},
{'name': 'messenger-api',
'credentials': [{'accessKey': api_id, 'secretKey': api_secret}],
'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]},
]}
write('s3_identities.json', json.dumps(identities, indent=2))
mq_conf = SECRETS / 'rabbitmq_auth.conf'
if mq_conf.exists():
mq = [l.split('=', 1)[1].strip() for l in mq_conf.read_text().splitlines() if l.startswith('default_pass')][0]
print('keep secrets/rabbitmq_auth.conf')
else:
mq = secrets.token_urlsafe(24)
write('rabbitmq_auth.conf', 'default_user = messenger\ndefault_pass = %s\ndefault_vhost = /' % mq)
write('broker_url', 'amqp://messenger:%s@rabbitmq:5672/' % mq)
env = ROOT / '.env'
if not env.exists():
shutil.copy(ROOT / '.env.example', env)
print('created .env from .env.example')
if __name__ == '__main__':
main()
+428
View File
@@ -0,0 +1,428 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, check
from image_probe import jpeg_size
import imagegen
from lablib import sh
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
NETWORK = 'mtusi-lab5_data'
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
def s3tool(*args):
cmd = ['docker', 'run', '--rm', '--network', NETWORK,
'-v', str(ROOT / 'secrets').replace('\\', '/') + ':/secrets:ro',
'-v', str(ROOT / 'scripts').replace('\\', '/') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
r = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('s3tool failed: ' + (r.stdout + r.stderr).strip()[-300:])
return json.loads(r.stdout.strip().splitlines()[-1])
def psql(sql):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-F', '|', '-v', 'ON_ERROR_STOP=1', '-c', sql)
def queues():
out = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues',
'name', 'messages_ready', 'messages_unacknowledged', 'consumers', '--formatter', 'json')
return {q['name']: {'ready': q['messages_ready'], 'unacked': q['messages_unacknowledged'],
'consumers': q['consumers']} for q in json.loads(out)}
def restarts(service):
out = sh('docker', 'compose', 'ps', '-a', '-q', service)
return {cid[:12]: int(sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid)) for cid in out.split()}
def events(att_id):
rows = psql("SELECT e.attempt, e.event, COALESCE(e.error_code,''), "
"to_char(e.created_at AT TIME ZONE 'UTC','HH24:MI:SS.MS') "
"FROM job_events e JOIN attachment_jobs j USING (job_id) "
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
return [dict(zip(('attempt', 'event', 'error_code', 'at'), r.split('|'))) for r in rows.splitlines()]
def event_times(att_id):
rows = psql("SELECT e.event, extract(epoch FROM e.created_at) FROM job_events e JOIN attachment_jobs j USING (job_id) "
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
return [(r.split('|')[0], float(r.split('|')[1])) for r in rows.splitlines()]
def job_row(att_id):
r = psql("SELECT j.job_id, j.status, j.attempt, j.max_attempts, COALESCE(j.error_code,''), "
"(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id), "
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id), "
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL), "
"a.staging_key IS NULL "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.attachment_id = '%s'" % att_id)
k = ('job_id', 'status', 'attempt', 'max_attempts', 'error_code', 'variant_rows', 'outbox_rows',
'outbox_unpublished', 'staging_removed')
v = r.split('|')
d = dict(zip(k, v))
for f in ('attempt', 'max_attempts', 'variant_rows', 'outbox_rows', 'outbox_unpublished'):
d[f] = int(d[f])
d['staging_removed'] = d['staging_removed'] == 't'
return d
def s3_counts(chat_id, att_id):
return {'variants_objects': s3tool('count', 'variants/%s/%s/' % (chat_id, att_id))['count'],
'final_original': s3tool('count', 'attachments/%s/%s' % (chat_id, att_id))['count'],
'staging_objects': s3tool('count', 'staging/' + att_id)['count']}
def upload(c, actor, chat, data=PNG, name='sample.png', ctype='image/png', expected=202):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
actor=actor, raw=data, headers={'Content-Type': ctype},
schema='Attachment' if expected == 202 else None)
def wait_status(c, actor, att_id, want, timeout=120):
deadline = time.monotonic() + timeout
seen = []
while time.monotonic() < deadline:
try:
a = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
except Failure:
time.sleep(0.5)
continue
if not seen or seen[-1] != a['status']:
seen.append(a['status'])
if a['status'] in want:
return a, seen
time.sleep(0.3)
raise Failure('attachment did not reach %s in %ss (seen %s)' % (want, timeout, seen))
def wait_api(c, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.5)
raise Failure('API not ready')
def check_ready(c, actor, att):
names = sorted(v['name'] for v in att['variants'])
check(names == ['crop', 'optimized', 'thumbnail'], 'variants: %s' % names)
orig = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)
check(hashlib.sha256(orig['bytes']).hexdigest() == att['sha256'], 'original changed')
out = {'variants': {}}
for v in att['variants']:
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=actor)
check(hashlib.sha256(r['bytes']).hexdigest() == v['sha256'], v['name'] + ' checksum')
check(r['headers'].get_content_type() == v['content_type'], v['name'] + ' content type')
out['variants'][v['name']] = '%s %dx%d %dB' % (v['content_type'], v['width'], v['height'], v['size_bytes'])
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=actor)['bytes']
out['thumbnail_real_jpeg_size'] = list(jpeg_size(thumb))
out['original_sha256_ok'] = True
return out
def fault_upload(c, actor, chat, point):
name = 'fault-%s-%d.png' % (point, int(time.time() * 1000))
try:
att = upload(c, actor, chat, name=name)['json']
return att['id'], name, 202
except Failure as exc:
att_id = psql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name))
return att_id, name, 'client saw failure: ' + str(exc)[-60:]
def cmd_worker_down(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
steps = []
sh('docker', 'compose', 'stop', 'worker')
steps.append('docker compose stop worker')
t0 = time.monotonic()
r = upload(c, alice, chat)
att = r['json']
upload_ms = round((time.monotonic() - t0) * 1000)
steps.append('upload -> %d %s' % (r['status'], att['status']))
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
steps.append('message with queued attachment -> 201')
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
steps.append('content while queued -> 409')
c.message(bob, chat, 'текст работает без worker')
steps.append('plain text message -> 201')
time.sleep(2)
q_before = queues()
job_before = job_row(att['id'])
sh('docker', 'compose', 'restart', 'api')
sh('docker', 'compose', 'restart', 'rabbitmq')
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 'rabbitmq', 'api', 'lb')
wait_api(c)
steps.append('docker compose restart api rabbitmq (volumes kept)')
q_after = queues()
still = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status']
sh('docker', 'compose', 'start', 'worker')
steps.append('docker compose start worker')
t1 = time.monotonic()
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
check(ready['status'] == 'ready', 'did not become ready')
res = {
'attachment_id': att['id'], 'message_id': msg['id'],
'upload_status': r['status'], 'upload_ms_worker_down': upload_ms,
'queue_before_restart': q_before.get('attachment.process.v1'),
'job_before_restart': job_before,
'queue_after_api_and_broker_restart': q_after.get('attachment.process.v1'),
'status_after_restart_before_worker': still,
'seconds_to_ready_after_worker_start': round(time.monotonic() - t1, 2),
'statuses_seen': seen, 'steps': steps,
'ready_check': check_ready(c, bob, ready),
'job_after': job_row(att['id']), 's3': s3_counts(chat['id'], att['id']),
}
return res
def cmd_api_crash(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
before = restarts('api')
att_id, name, client_view = fault_upload(c, alice, chat, 'api-after-commit')
check(att_id, 'attachment row not found')
row = job_row(att_id)
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'not ready')
time.sleep(3)
return {'fault': 'API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response',
'attachment_id': att_id, 'client_view': client_view,
'job_right_after_crash': row, 'api_restart_counts_before': before, 'api_restart_counts_after': restarts('api'),
'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id),
'ready_check': check_ready(c, bob, ready)}
def _worker_crash(a, point, desc):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
before = restarts('worker')
att_id, name, client_view = fault_upload(c, alice, chat, point)
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'not ready')
time.sleep(5)
job = job_row(att_id)
s3 = s3_counts(chat['id'], att_id)
check(job['variant_rows'] == 3 and s3['variants_objects'] == 3, 'duplicated or missing variants')
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '3m')
noop = [json.loads(l) for l in logs.splitlines() if att_id in l and 'no-op' in l]
return {'fault': desc, 'attachment_id': att_id, 'upload': client_view,
'worker_restart_counts_before': before, 'worker_restart_counts_after': restarts('worker'),
'statuses_seen': seen, 'events': events(att_id), 'job_after': job, 's3': s3,
'redelivery_noop_log': [{'result': n.get('result'), 'redelivered': n.get('redelivered')} for n in noop],
'ready_check': check_ready(c, bob, ready)}
def cmd_effect_crash(a):
return _worker_crash(a, 'worker-after-effect',
'worker os.Exit after writing original+variants to S3, before result COMMIT and ack')
def cmd_ack_crash(a):
return _worker_crash(a, 'worker-before-ack', 'worker os.Exit after result COMMIT + staging cleanup, before ack')
def cmd_retry(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
att_id, _, _ = fault_upload(c, alice, chat, 'transient')
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'not ready')
t = event_times(att_id)
fails = [x for x in t if x[0] == 'retry_scheduled']
starts = [x for x in t if x[0] == 'started']
gaps = [round(starts[i + 1][1] - fails[i][1], 2) for i in range(len(fails))]
return {'fault': 'attempts 1 and 2 raise a transient error (injected_transient)',
'attachment_id': att_id, 'statuses_seen': seen, 'events': events(att_id),
'retry_gaps_seconds': gaps,
'expected_gap_ranges': ['attempt1: backoff [0.5, 1.0] s + relay poll <= 1 s',
'attempt2: backoff [1.0, 2.0] s + relay poll <= 1 s'],
'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)}
def cmd_replay(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
att_id, _, _ = fault_upload(c, alice, chat, 'flaky')
failed, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=180)
check(failed['status'] == 'failed', 'expected failed')
res = {'fault': 'transient error on every attempt <= JOB_MAX_ATTEMPTS', 'attachment_id': att_id,
'statuses_seen': seen, 'failed_metadata': {k: failed[k] for k in ('status', 'error_code', 'variants')},
'content_when_failed': c.call('GET', '/api/v1/attachments/' + att_id + '/content', 409, actor=bob)['status'],
'attach_failed_to_message': c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text': 'x', 'attachment_ids': [att_id]},
headers={'Idempotency-Key': 'replay-' + att_id})['status'],
'job_failed': job_row(att_id)}
res['dlq_registry_query'] = psql("SELECT count(*) FROM attachment_jobs WHERE status = 'failed' AND attachment_id = '%s'" % att_id)
replay = lambda: json.loads(sh('docker', 'compose', 'exec', '-T', 'worker', '/app/server', 'replay', att_id,
'--actor', 'lab5ops'))
res['replay_1'] = replay()
ready, seen2 = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'replay did not finish')
res['replay_2_after_ready'] = replay()
res['statuses_after_replay'] = seen2
res['events'] = events(att_id)
res['job_after'] = job_row(att_id)
res['ready_check'] = check_ready(c, bob, ready)
return res
def corrupt_png():
good = imagegen.png(64, 64)
return (b'\x89PNG\r\n\x1a\n' + imagegen.chunk(b'IHDR', good[16:29]) +
imagegen.chunk(b'IDAT', b'not-a-zlib-stream' * 10) + imagegen.chunk(b'IEND', b''))
def cmd_invalid(a):
c = Client(a.base_url, timeout=60)
alice, bob, _eve, chat = c.bootstrap()
att = upload(c, alice, chat, data=corrupt_png(), name='corrupt.png')['json']
failed, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image', 'expected invalid_image')
return {
'corrupt_png': {'upload': 202, 'statuses_seen': seen, 'error_code': failed['error_code'],
'events': events(att['id']), 'job': job_row(att['id'])},
'api_rejects_before_queue': {
'pixel_bomb_20000x20000': upload(c, alice, chat, data=imagegen.bomb_png(), name='bomb.png', expected=413)['status'],
'not_an_image': upload(c, alice, chat, data=b'GIF89a' + b'\0' * 100, name='x.png', expected=415)['status'],
'over_10MiB': upload(c, alice, chat, data=b'\x89PNG\r\n\x1a\n' + b'\0' * (10 * 1024 * 1024 + 1),
name='big.png', expected=413)['status'],
'path_in_filename': upload(c, alice, chat, name='..%2Fetc.png', expected=400)['status'],
}}
def cmd_hang(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
att_id, _, _ = fault_upload(c, alice, chat, 'hang')
t0 = time.monotonic()
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=150)
check(ready['status'] == 'ready', 'not ready')
took = round(time.monotonic() - t0, 1)
time.sleep(70)
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '5m')
stale = [l for l in logs.splitlines() if att_id in l and ('stale attempt' in l or 'no-op' in l)]
return {'fault': 'attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)',
'attachment_id': att_id, 'seconds_to_ready': took, 'statuses_seen': seen,
'events': events(att_id), 'job_after': job_row(att_id), 's3': s3_counts(chat['id'], att_id),
'stale_attempt_logs': [json.loads(l).get('msg') for l in stale],
'ready_check': check_ready(c, bob, ready)}
def cmd_dlq(a):
before = queues()
script = ('U=$(sed -n "s/^default_user = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
'P=$(sed -n "s/^default_pass = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
'wget -q -O- --header "Content-Type: application/json" --post-data '
'\'{"properties":{"delivery_mode":2},"routing_key":"attachment.process.v1",'
'"payload":"{\\"not\\":\\"a job\\"}","payload_encoding":"string"}\' '
'"http://$U:$P@127.0.0.1:15672/api/exchanges/%2F/attachments/publish"')
routed = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'sh', '-c', script)
time.sleep(3)
after = queues()
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '1m')
return {'published': json.loads(routed), 'dlq_before': before.get('attachment.process.dlq'),
'dlq_after': after.get('attachment.process.dlq'),
'worker_log': [json.loads(l)['msg'] for l in logs.splitlines() if 'DLQ' in l][-1:]}
def gradient_png(w, h):
row = b'\x02' + bytes(3 * w)
first = b'\x00' + bytes((x * 255 // max(1, w - 1)) for x in range(w) for _ in range(3))
raw = first + row * (h - 1)
return imagegen.png(w, h, raw=raw, level=9)
class MemSampler:
def __init__(self, service):
self.cids = sh('docker', 'compose', 'ps', '-q', service).split()
self.peak = 0.0
self._stop = threading.Event()
self.t = threading.Thread(target=self.run, daemon=True)
def run(self):
while not self._stop.is_set():
out = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *self.cids, check=False)
for line in out.splitlines():
used = line.split('/')[0].strip()
num = float(''.join(ch for ch in used if ch.isdigit() or ch == '.') or 0)
mib = num * 1024 if used.endswith('GiB') else num / 1024 if used.endswith('KiB') else num
self.peak = max(self.peak, mib)
def __enter__(self):
self.t.start()
return self
def __exit__(self, *exc):
self._stop.set()
self.t.join(timeout=10)
def cmd_limits(a):
c = Client(a.base_url, timeout=60)
alice, bob, _eve, chat = c.bootstrap()
data = gradient_png(6000, 6000)
limit = sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 'worker').split()[0])
with MemSampler('worker') as m:
t0 = time.monotonic()
att = upload(c, alice, chat, data=data, name='big-6000x6000.png')['json']
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120)
took = round(time.monotonic() - t0, 1)
time.sleep(2)
check(ready['status'] == 'ready', 'big image failed: %s' % ready['error_code'])
return {'image': '6000x6000 PNG (36 MPx, %d bytes)' % len(data), 'seconds_to_ready': took,
'worker_peak_mem_mib': round(m.peak, 1), 'worker_mem_limit_mib': int(limit) // (1 << 20),
'variants': {v['name']: '%dx%d' % (v['width'], v['height']) for v in ready['variants']},
'statuses_seen': seen, 'job': job_row(att['id'])}
COMMANDS = {
'worker-down': cmd_worker_down, 'api-crash': cmd_api_crash, 'effect-crash': cmd_effect_crash,
'ack-crash': cmd_ack_crash, 'retry': cmd_retry, 'replay': cmd_replay, 'invalid': cmd_invalid,
'hang': cmd_hang, 'dlq': cmd_dlq, 'limits': cmd_limits,
}
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('command', choices=sorted(COMMANDS))
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
a = p.parse_args()
started = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
try:
res = COMMANDS[a.command](a)
res = {'scenario': a.command, 'result': 'PASS', 'started_at': started, **res}
except (Failure, RuntimeError) as exc:
res = {'scenario': a.command, 'result': 'FAIL', 'started_at': started, 'error': str(exc)}
text = json.dumps(res, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
if res['result'] != 'PASS':
raise SystemExit(1)
if __name__ == '__main__':
main()
+130
View File
@@ -0,0 +1,130 @@
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
PROJECT = 'mtusi-lab5'
def sh(*args, check=True):
r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
if check and r.returncode != 0:
raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300]))
return r.stdout.strip()
def api_containers(all_states=False):
out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc',
'--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api',
'--format', '{{json .}}')
rows = []
for line in out.splitlines():
d = json.loads(line)
rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']})
return sorted(rows, key=lambda r: r['name'])
def health(cid):
return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid)
def wait_healthy(cid, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if health(cid) == 'healthy':
return True
time.sleep(0.25)
return False
def ps_snapshot():
return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}')
class WriteLoad:
def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0):
self.base = base_url
self.workers = workers
self.timeout = timeout
self.pause = pause
self.events = []
self.confirmed = []
self.retries = 0
self._lock = threading.Lock()
self._stop = threading.Event()
self._threads = []
c = Client(base_url, timeout=10)
self.alice, self.bob, _eve, self.chat = c.bootstrap()
self.reader = c
def _worker(self, wid):
c = Client(self.base, timeout=self.timeout)
key = text = None
n = 0
while not self._stop.is_set():
if key is None:
key = uuid.uuid4().hex
text = 'load-%d-%d-%s' % (wid, n, key[:8])
start = time.monotonic()
try:
r = c.message(self.alice, self.chat, text, key=key)
with self._lock:
self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start))
self.confirmed.append(r['json'])
key = None
n += 1
except Failure as exc:
with self._lock:
self.events.append((start, False, str(exc)[-60:], time.monotonic() - start))
self.retries += 1
time.sleep(0.05)
if self.pause:
time.sleep(self.pause)
def start(self):
for i in range(self.workers):
t = threading.Thread(target=self._worker, args=(i,), daemon=True)
t.start()
self._threads.append(t)
self.t0 = time.monotonic()
def stop(self):
self._stop.set()
for t in self._threads:
t.join(timeout=15)
def window(self, a, b):
ev = [e for e in self.events if a <= e[0] < b]
ok = [e for e in ev if e[1]]
lat = sorted(e[3] for e in ok)
pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None
return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok),
'ok_rps': round(len(ok) / (b - a), 1) if b > a else None,
'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)}
def verify_history(self):
items, cursor = [], None
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100'
while True:
page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''),
actor=self.bob, schema='MessagePage')['json']
items += page['items']
cursor = page['next_cursor']
if not cursor:
break
by_id = {m['id']: m for m in items}
texts = [m['text'] for m in items]
lost = [m for m in self.confirmed if by_id.get(m['id']) != m]
return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed),
'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id),
'duplicate_texts': len(texts) - len(set(texts)),
'unique_confirmed': len({m['id'] for m in self.confirmed})}
+6
View File
@@ -0,0 +1,6 @@
@echo off
cd /d "%~dp0.."
docker compose stop lb api worker || exit /b 1
docker compose restart postgres valkey s3 rabbitmq || exit /b 1
docker compose up -d --no-recreate --wait postgres valkey s3 rabbitmq || exit /b 1
docker compose start api worker lb || exit /b 1
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose stop lb api worker
docker compose restart postgres valkey s3 rabbitmq
docker compose up -d --no-recreate --wait postgres valkey s3 rabbitmq
docker compose start api worker lb
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
import argparse
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import hmac
import http.client
import json
import os
import sys
import threading
from urllib.parse import quote
import xml.etree.ElementTree as ET
HOST = os.environ.get('S3_HOST', 's3:8333')
REGION = os.environ.get('S3_REGION', 'us-east-1')
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
def creds(identity):
if identity == 'anonymous':
return None
prefix = 's3_admin_' if identity == 'admin' else 's3_'
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
return read('access_key_id'), read('secret_access_key')
def _h(key, msg):
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
_local = threading.local()
def request(method, path, identity='api', body=b'', query=None):
query = query or {}
headers = {'host': HOST}
payload_hash = hashlib.sha256(body).hexdigest()
c = creds(identity)
if c:
now = dt.datetime.now(dt.timezone.utc)
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
signed = ';'.join(sorted(headers))
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
scope = '%s/%s/s3/aws4_request' % (date, REGION)
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
c[0], scope, signed, sig)
url = quote(path, safe='/-_.~')
if query:
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
conn = getattr(_local, 'conn', None)
if conn is None:
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
try:
conn.request(method, url, body=body or None, headers=headers)
r = conn.getresponse()
return r.status, r.read()
except (OSError, http.client.HTTPException):
_local.conn = None
raise
def obj(key):
return '/' + BUCKET + '/' + key
def list_keys(prefix, identity='api'):
keys, token = [], None
while True:
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
if token:
q['continuation-token'] = token
st, body = request('GET', '/' + BUCKET, identity, query=q)
if st != 200:
raise SystemExit('list failed: HTTP %d' % st)
root = ET.fromstring(body)
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
if root.findtext(NS + 'IsTruncated') != 'true':
return keys
token = root.findtext(NS + 'NextContinuationToken')
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
p.add_argument('arg', nargs='?', default='')
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
p.add_argument('--size', type=int, default=100)
p.add_argument('--workers', type=int, default=32)
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
a = p.parse_args()
out = {'cmd': a.cmd, 'as': a.identity}
if a.cmd == 'get':
st, body = request('GET', obj(a.arg), a.identity)
out.update(status=st, bytes=len(body) if st == 200 else 0,
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
elif a.cmd == 'put':
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
out.update(status=st)
elif a.cmd == 'delete':
st, _ = request('DELETE', obj(a.arg), a.identity)
out.update(status=st)
elif a.cmd == 'create-bucket':
st, _ = request('PUT', '/' + a.arg, a.identity)
out.update(status=st)
elif a.cmd == 'list':
out.update(keys=list_keys(a.arg, a.identity))
elif a.cmd == 'count':
out.update(count=len(list_keys(a.arg, a.identity)))
elif a.cmd in ('bulk-put', 'bulk-delete'):
keys = [k.strip() for k in sys.stdin if k.strip()]
data = b'x' * a.size
def one(k):
if a.cmd == 'bulk-put':
return request('PUT', obj(k), a.identity, body=data)[0]
return request('DELETE', obj(k), a.identity)[0]
start = dt.datetime.now()
with ThreadPoolExecutor(a.workers) as pool:
codes = list(pool.map(one, keys))
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
print(json.dumps(out))
if __name__ == '__main__':
main()
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
docker compose start worker
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose start worker
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
set CID=
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api"') do set CID=%%i
if "%CID%"=="" exit /b 3
docker stop -t 15 %CID% >nul || exit /b 1
echo stopped api container %CID%
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;;
esac
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=mtusi-lab5" \
--filter "label=com.docker.compose.service=api")
[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; }
[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; }
docker stop -t 15 "$cid" >/dev/null
echo "stopped api container $cid"
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
docker compose stop worker
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose stop worker
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
sys.path.insert(0, str(ROOT / 'student-tests'))
from client import Client, Failure
from lab5ops import MemSampler, gradient_png, psql, queues
import imagegen
def compose(*args, env=None):
e = dict(os.environ)
e.update(env or {})
r = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, env=e,
capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('compose %s: %s' % (args[:2], r.stderr[-300:]))
def noisy_png(w, h):
row_noise = lambda: b'\x00' + os.urandom(3 * w)
row_same = b'\x02' + bytes(3 * w)
raw = b''.join(row_noise() if y % 8 == 0 else row_same for y in range(h))
return imagegen.png(w, h, raw=raw, level=6)
SIZES = {
'S_320x200': lambda: (ROOT / 'tests/fixtures/sample.png').read_bytes(),
'M_2000x1500': lambda: noisy_png(2000, 1500),
'L_5000x4000': lambda: gradient_png(5000, 4000),
}
COUNTS = {'S_320x200': 120, 'M_2000x1500': 40, 'L_5000x4000': 16}
CONCURRENCY = [1, 2, 4]
def pct(values, p):
if not values:
return None
s = sorted(values)
return round(s[min(len(s) - 1, int(p * len(s)))], 3)
class LagSampler(threading.Thread):
def __init__(self, chat_id):
super().__init__(daemon=True)
self.chat_id = chat_id
self.samples = []
self.stop = threading.Event()
def run(self):
t0 = time.monotonic()
while not self.stop.is_set():
try:
r = psql("SELECT count(*) FILTER (WHERE j.status IN ('queued','processing')), "
"COALESCE(extract(epoch FROM now() - min(j.created_at) FILTER (WHERE j.status IN ('queued','processing'))), 0) "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % self.chat_id)
active, oldest = r.split('|')
q = queues().get('attachment.process.v1', {})
self.samples.append({'t': round(time.monotonic() - t0, 1), 'active': int(active),
'mq_ready': q.get('ready'), 'mq_unacked': q.get('unacked'),
'oldest_age_s': round(float(oldest), 2)})
except Exception:
pass
self.stop.wait(0.5)
def burst(c, actor, chat, data, n, threads=8):
codes = []
lock = threading.Lock()
def one(i):
try:
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=t1-%d.png' % i, [202, 503],
actor=actor, raw=data, headers={'Content-Type': 'image/png'})
code = r['status']
ra = r['headers'].get('Retry-After')
except Failure as exc:
code, ra = 'error:' + str(exc)[-40:], None
with lock:
codes.append((code, ra))
t0 = time.monotonic()
with ThreadPoolExecutor(threads) as pool:
list(pool.map(one, range(n)))
return codes, time.monotonic() - t0
def wait_drained(chat_id, timeout=900):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
n = int(psql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
"WHERE a.chat_id = '%s' AND j.status IN ('queued','processing')" % chat_id))
if n == 0:
return True
time.sleep(1)
return False
def job_stats(chat_id):
rows = psql("SELECT j.status, extract(epoch FROM j.created_at), extract(epoch FROM j.finished_at), j.attempt "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % chat_id)
jobs = [r.split('|') for r in rows.splitlines()]
ready = [(float(c), float(f)) for s, c, f, _ in jobs if s == 'ready']
ttr = [f - c for c, f in ready]
span = (max(f for _, f in ready) - min(c for c, _ in ready)) if ready else 0
return {'jobs': len(jobs), 'ready': len(ready), 'failed': sum(1 for j in jobs if j[0] == 'failed'),
'retried_jobs': sum(1 for j in jobs if int(j[3]) > 1),
'throughput_jobs_per_s': round(len(ready) / span, 2) if span else None,
'pipeline_span_s': round(span, 2),
'time_to_ready_s': {'p50': pct(ttr, .5), 'p95': pct(ttr, .95), 'max': round(max(ttr), 3) if ttr else None}}
def set_worker(concurrency):
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_CONCURRENCY': str(concurrency)})
time.sleep(2)
def one_run(base_url, size, conc):
c = Client(base_url, timeout=60)
alice, _bob, _eve, chat = c.bootstrap()
data = SIZES[size]()
n = COUNTS[size]
lag = LagSampler(chat['id'])
with MemSampler('worker') as mem:
lag.start()
codes, upload_s = burst(c, alice, chat, data, n)
drained = wait_drained(chat['id'])
time.sleep(1)
lag.stop.set()
lag.join()
st = job_stats(chat['id'])
s = lag.samples
return {'size': size, 'image_bytes': len(data), 'worker_concurrency_prefetch': conc, 'uploads': n,
'accepted_202': sum(1 for x, _ in codes if x == 202),
'rejected': sum(1 for x, _ in codes if x != 202),
'upload_burst_s': round(upload_s, 2), 'drained': drained, **st,
'queue_lag': {'max_active_jobs': max((x['active'] for x in s), default=None),
'max_mq_ready': max((x['mq_ready'] or 0 for x in s), default=None),
'max_oldest_job_age_s': max((x['oldest_age_s'] for x in s), default=None)},
'worker_peak_rss_mib': round(mem.peak, 1),
'lag_timeline': s[::max(1, len(s) // 40)]}
def cmd_matrix(a):
runs = []
try:
for conc in CONCURRENCY:
set_worker(conc)
for size in SIZES:
r = one_run(a.base_url, size, conc)
print(json.dumps({k: r[k] for k in ('size', 'worker_concurrency_prefetch', 'throughput_jobs_per_s',
'time_to_ready_s', 'worker_peak_rss_mib', 'queue_lag')}), flush=True)
runs.append(r)
finally:
set_worker(2)
return {'runs': runs}
def cmd_backpressure(a):
bound = 20
compose('up', '-d', '--no-deps', '--wait', 'api', env={'MAX_ACTIVE_JOBS': str(bound)})
set_worker(1)
try:
c = Client(a.base_url, timeout=60)
alice, _bob, _eve, chat = c.bootstrap()
data = SIZES['M_2000x1500']()
lag = LagSampler(chat['id'])
lag.start()
t0 = time.monotonic()
codes, upload_s = burst(c, alice, chat, data, 80)
drained = wait_drained(chat['id'])
drain_s = time.monotonic() - t0
t1 = time.monotonic()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after.png', 202, actor=alice,
raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
while True:
st = c.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']['status']
if st in ('ready', 'failed'):
break
time.sleep(0.2)
recovery_ready_s = time.monotonic() - t1
time.sleep(1)
lag.stop.set()
lag.join()
s = lag.samples
return {'max_active_jobs_bound': bound, 'worker_concurrency': 1, 'uploads': 80,
'accepted_202': sum(1 for x, _ in codes if x == 202),
'rejected_503': sum(1 for x, _ in codes if x == 503),
'retry_after_header': sorted({ra for x, ra in codes if x == 503}),
'other': [x for x, _ in codes if x not in (202, 503)],
'upload_burst_s': round(upload_s, 2), 'drained': drained, 'drain_s': round(drain_s, 1),
'max_active_observed': max((x['active'] for x in s), default=None),
'jobs': job_stats(chat['id']), 'after_overload_upload_status': st,
'after_overload_seconds_to_ready': round(recovery_ready_s, 2),
'lag_timeline': s[::max(1, len(s) // 40)]}
finally:
compose('up', '-d', '--no-deps', '--wait', 'api')
set_worker(2)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('command', choices=['matrix', 'backpressure'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
a = p.parse_args()
env = {'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'worker_limits': 'cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB',
'host': 'Windows 11, Docker Desktop 29.7.2 (WSL2)'}
res = {'experiment': a.command, **env, **(cmd_matrix(a) if a.command == 'matrix' else cmd_backpressure(a))}
text = json.dumps(res, ensure_ascii=False, indent=1)
if a.output:
(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
print(text[:3000])
if __name__ == '__main__':
main()
+403
View File
@@ -0,0 +1,403 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"mtusi-messenger/internal/api"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/cache"
"mtusi-messenger/internal/db"
"mtusi-messenger/internal/imaging"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/relay"
"mtusi-messenger/internal/store"
"mtusi-messenger/internal/worker"
"golang.org/x/sync/semaphore"
)
func getenv(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
func getint(key string, def int) int {
v := os.Getenv(key)
if v == "" {
return def
}
n, err := strconv.Atoi(v)
if err != nil || n <= 0 {
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
os.Exit(2)
}
return n
}
func secret(key string) (string, error) {
if path := os.Getenv(key + "_FILE"); path != "" {
b, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read %s_FILE: %w", key, err)
}
return strings.TrimSpace(string(b)), nil
}
if v := os.Getenv(key); v != "" {
return v, nil
}
return "", fmt.Errorf("%s or %s_FILE is required", key, key)
}
func main() {
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
if err := run(log); err != nil {
log.Error("fatal", "error", err.Error())
os.Exit(1)
}
}
func openDB(ctx context.Context) (*store.Store, func(), error) {
dsn, err := secret("DATABASE_URL")
if err != nil {
return nil, nil, err
}
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))})
if err != nil {
return nil, nil, errors.New("database config is invalid")
}
return store.New(pool), pool.Close, nil
}
func getduration(key string, def time.Duration) time.Duration {
v := os.Getenv(key)
if v == "" {
return def
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
os.Exit(2)
}
return d
}
func openObjects() (*objstore.Store, error) { return openObjectsAs("S3") }
func openObjectsAs(prefix string) (*objstore.Store, error) {
access, err := secret(prefix + "_ACCESS_KEY_ID")
if err != nil {
return nil, err
}
secretKey, err := secret(prefix + "_SECRET_ACCESS_KEY")
if err != nil {
return nil, err
}
return objstore.New(objstore.Config{
Endpoint: getenv("S3_ENDPOINT", "http://s3:9000"),
Region: getenv("S3_REGION", "us-east-1"),
Bucket: getenv("S3_BUCKET", "messenger-attachments"),
AccessKey: access,
SecretKey: secretKey,
})
}
func getbool(key string) bool {
v := strings.ToLower(os.Getenv(key))
return v == "1" || v == "true" || v == "yes"
}
func startRelay(ctx context.Context, st *store.Store, log *slog.Logger) (*relay.Relay, func(), error) {
url, err := secret("BROKER_URL")
if err != nil {
return nil, nil, err
}
pub := broker.NewPublisher(url)
rl := relay.New(st, pub, log, getduration("OUTBOX_POLL_INTERVAL", time.Second))
rctx, cancel := context.WithCancel(ctx)
go rl.Run(rctx)
return rl, func() { cancel(); pub.Close() }, nil
}
func runWorker(ctx context.Context, log *slog.Logger) error {
st, closeDB, err := openDB(ctx)
if err != nil {
return err
}
defer closeDB()
objects, err := openObjects()
if err != nil {
return err
}
url, err := secret("BROKER_URL")
if err != nil {
return err
}
budget := int64(getint("WORKER_DECODE_BUDGET_BYTES", 384<<20))
cfg := worker.Config{
BrokerURL: url,
Concurrency: getint("WORKER_CONCURRENCY", 2),
MaxAttempts: getint("JOB_MAX_ATTEMPTS", 5),
JobTimeout: time.Duration(getint("JOB_TIMEOUT_SECONDS", 60)) * time.Second,
Lease: time.Duration(getint("JOB_LEASE_SECONDS", 30)) * time.Second,
BackoffBase: getduration("JOB_BACKOFF_BASE", time.Second),
BackoffMax: getduration("JOB_BACKOFF_MAX", time.Minute),
StagingRetain: time.Duration(getint("STAGING_RETENTION_SECONDS", 86400)) * time.Second,
PendingGrace: getduration("PENDING_UPLOAD_GRACE", 15*time.Minute),
FaultInjection: getbool("FAULT_INJECTION"),
SweepEvery: getduration("SWEEP_INTERVAL", 5*time.Second),
Limits: imaging.Limits{
MaxBytes: int64(getint("MAX_UPLOAD_BYTES", 10485760)),
MaxPixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)),
Budget: semaphore.NewWeighted(budget),
BudgetSize: budget,
},
}
if cfg.Lease < 6*time.Second {
return errors.New("JOB_LEASE_SECONDS must be >= 6")
}
wctx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT)
defer stop()
if !getbool("WORKER_NO_RELAY") {
_, stopRelay, err := startRelay(wctx, st, log)
if err != nil {
return err
}
defer stopRelay()
}
log.Info("worker started", "concurrency", cfg.Concurrency, "max_attempts", cfg.MaxAttempts,
"job_timeout_s", int(cfg.JobTimeout.Seconds()), "lease_s", int(cfg.Lease.Seconds()),
"fault_injection", cfg.FaultInjection, "build", getenv("APP_BUILD", "dev"))
return worker.New(cfg, st, objects, log).Run(wctx)
}
func runReplay(ctx context.Context, args []string) error {
fs := flag.NewFlagSet("replay", flag.ContinueOnError)
actor := fs.String("actor", "operator", "who requested the replay (audit)")
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
args = append(args[1:], args[0])
}
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() != 1 {
return errors.New("usage: replay <attachment_id> [--actor name]")
}
st, closeDB, err := openDB(ctx)
if err != nil {
return err
}
defer closeDB()
jobID, attempt, err := st.ReplayFailed(ctx, fs.Arg(0), *actor, getint("JOB_MAX_ATTEMPTS", 5))
switch {
case errors.Is(err, store.ErrJobAlreadyOK):
fmt.Println(`{"result":"noop","reason":"job already ready; replay refused"}`)
return nil
case errors.Is(err, store.ErrJobNotFailed):
fmt.Println(`{"result":"noop","reason":"job is queued/processing, not failed"}`)
return nil
case errors.Is(err, store.ErrStagingGone):
fmt.Println(`{"result":"refused","reason":"staging original removed by retention"}`)
return nil
case err != nil:
return err
}
out, _ := json.Marshal(map[string]any{"result": "replayed", "job_id": jobID, "attempt": attempt})
fmt.Println(string(out))
return nil
}
func run(log *slog.Logger) error {
ctx := context.Background()
if len(os.Args) > 1 && os.Args[1] == "migrate" {
dsn, err := secret("DATABASE_URL")
if err != nil {
return err
}
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2})
if err != nil {
return errors.New("database config is invalid")
}
defer pool.Close()
mctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
return db.Migrate(mctx, pool, log)
}
if len(os.Args) > 1 && os.Args[1] == "s3-init" {
objects, err := openObjectsAs("S3_ADMIN")
if err != nil {
return err
}
var created bool
for attempt := 1; ; attempt++ {
bctx, cancel := context.WithTimeout(ctx, 5*time.Second)
created, err = objects.EnsureBucket(bctx, getenv("S3_REGION", "us-east-1"))
cancel()
if err == nil || attempt == 30 {
break
}
log.Warn("s3 not ready, retrying", "attempt", attempt, "error", err.Error())
time.Sleep(2 * time.Second)
}
if err != nil {
return err
}
log.Info("bucket ready", "bucket", objects.Bucket(), "created", created)
return nil
}
if len(os.Args) > 1 && os.Args[1] == "worker" {
return runWorker(ctx, log)
}
if len(os.Args) > 1 && os.Args[1] == "replay" {
return runReplay(ctx, os.Args[2:])
}
st, closeDB, err := openDB(ctx)
if err != nil {
return err
}
defer closeDB()
ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second
cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second
var sessCache auth.SessionCache
switch mode := getenv("SESSION_STORE", "postgres"); mode {
case "postgres":
case "valkey", "redis":
url, err := secret("CACHE_URL")
if err != nil {
return err
}
vk, err := cache.NewValkey(url)
if err != nil {
return errors.New("cache config is invalid")
}
defer vk.Close()
sessCache = vk
default:
return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode)
}
hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4))
sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log)
instanceID := getenv("INSTANCE_ID", "")
if instanceID == "" {
h, err := os.Hostname()
if err != nil || h == "" {
return errors.New("cannot determine instance id; set INSTANCE_ID")
}
instanceID = h
}
idemTTL := time.Duration(getint("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second
objects, err := openObjects()
if err != nil {
return err
}
maxUpload := int64(getint("MAX_UPLOAD_BYTES", 10485760))
if maxUpload > 10485760 {
return errors.New("MAX_UPLOAD_BYTES above the 10 MiB contract limit")
}
drainDelay := time.Duration(getint("SHUTDOWN_DRAIN_MS", 3000)) * time.Millisecond
rl, stopRelay, err := startRelay(ctx, st, log)
if err != nil {
return err
}
defer stopRelay()
ready := &atomic.Bool{}
ready.Store(true)
host := getenv("APP_HOST", "0.0.0.0")
port := getenv("APP_PORT", "8080")
addr := host + ":" + port
srv := api.NewServer(st, objects, sessions, hasher, log, ready, api.Options{
InstanceID: instanceID, IdempotencyTTL: idemTTL,
MaxUploadBytes: maxUpload,
MaxImagePixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)),
UploadConcurrency: int64(getint("UPLOAD_CONCURRENCY", 8)),
UploadTimeout: getduration("UPLOAD_TIMEOUT", 60*time.Second),
S3Timeout: getduration("S3_TIMEOUT", 15*time.Second),
Relay: rl,
MaxActiveJobs: getint("MAX_ACTIVE_JOBS", 500),
PipelineVersion: imaging.PipelineVersion,
MaxAttempts: getint("JOB_MAX_ATTEMPTS", 5),
FaultInjection: getbool("FAULT_INJECTION"),
})
httpSrv := &http.Server{
Addr: addr,
Handler: srv.Handler(),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 90 * time.Second,
WriteTimeout: 90 * time.Second,
IdleTimeout: 60 * time.Second,
}
bgCtx, stopBg := context.WithCancel(ctx)
defer stopBg()
go func() {
t := time.NewTicker(10 * time.Minute)
defer t.Stop()
for {
select {
case <-bgCtx.Done():
return
case <-t.C:
c, cancel := context.WithTimeout(bgCtx, 10*time.Second)
if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 {
log.Info("expired sessions purged", "count", n)
}
if n, err := st.PurgeIdempotency(c, idemTTL); err == nil && n > 0 {
log.Info("expired idempotency keys purged", "count", n)
}
cancel()
}
}
}()
errCh := make(chan error, 1)
go func() { errCh <- httpSrv.ListenAndServe() }()
log.Info("server started", "addr", addr, "pid", os.Getpid(), "instance_id", instanceID, "build", getenv("APP_BUILD", "dev"), "s3_bucket", objects.Bucket(),
"session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds()))
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
select {
case err := <-errCh:
if err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
case sig := <-stop:
log.Info("shutdown signal received", "signal", sig.String())
ready.Store(false)
time.Sleep(drainDelay)
httpSrv.SetKeepAlivesEnabled(false)
sctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := httpSrv.Shutdown(sctx); err != nil {
log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error())
_ = httpSrv.Close()
}
log.Info("server stopped cleanly")
}
return nil
}
+35
View File
@@ -0,0 +1,35 @@
module mtusi-messenger
go 1.23
require (
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.7.2
github.com/minio/minio-go/v7 v7.0.82
github.com/redis/go-redis/v9 v9.7.0
golang.org/x/crypto v0.31.0
golang.org/x/sync v0.10.0
)
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.8.1 // indirect
github.com/gen2brain/webp v0.5.2 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tetratelabs/wazero v1.8.1 // indirect
golang.org/x/image v0.23.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
)
+72
View File
@@ -0,0 +1,72 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE=
github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/gen2brain/webp v0.5.2 h1:aYdjbU/2L98m+bqUdkYMOIY93YC+EN3HuZLMaqgMD9U=
github.com/gen2brain/webp v0.5.2/go.mod h1:Nb3xO5sy6MeUAHhru9H3GT7nlOQO5dKRNNlE92CZrJw=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+334
View File
@@ -0,0 +1,334 @@
package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/google/uuid"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/store"
)
const (
mimePNG = "image/png"
mimeJPEG = "image/jpeg"
maxDrainBytes = 64 << 20
)
func (s *Server) objectError(w http.ResponseWriter, r *http.Request, err error) {
s.log.Warn("object storage unavailable", "request_id", requestID(r), "error", err.Error())
apperr.Write(w, requestID(r), apperr.Unavailable("attachment storage temporarily unavailable"))
}
func validFilename(name string) bool {
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || name == "." || name == ".." {
return false
}
if n := runeLen(name); n < 1 || n > 255 {
return false
}
for _, c := range name {
switch {
case c == '/', c == '\\', c == 0, unicode.IsControl(c):
return false
case c >= 0x202a && c <= 0x202e, c >= 0x2066 && c <= 0x2069:
return false
}
}
return true
}
func sniffImage(data []byte) string {
switch {
case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")):
return mimePNG
case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}):
return mimeJPEG
}
return ""
}
type imageInfo struct {
contentType string
width, height int
}
func (s *Server) inspectImage(data []byte, declared string) (imageInfo, *apperr.AppError) {
detected := sniffImage(data)
if detected == "" {
return imageInfo{}, apperr.UnsupportedMedia("content is not a PNG or JPEG image")
}
if detected != declared {
return imageInfo{}, apperr.UnsupportedMedia("Content-Type does not match the image content")
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || (format != "png" && format != "jpeg") {
return imageInfo{}, apperr.UnsupportedMedia("image header is invalid")
}
pixels := int64(cfg.Width) * int64(cfg.Height)
if pixels > s.maxPixels {
return imageInfo{}, apperr.PayloadTooLarge("image exceeds " + strconv.FormatInt(s.maxPixels, 10) + " pixels")
}
return imageInfo{contentType: detected, width: cfg.Width, height: cfg.Height}, nil
}
func readBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, *apperr.AppError) {
var buf bytes.Buffer
if r.ContentLength > 0 {
buf.Grow(int(r.ContentLength))
}
_, err := buf.ReadFrom(http.MaxBytesReader(w, r.Body, limit))
var maxErr *http.MaxBytesError
switch {
case errors.As(err, &maxErr):
return nil, apperr.PayloadTooLarge("attachment exceeds " + strconv.FormatInt(limit, 10) + " bytes")
case err != nil:
return nil, apperr.InvalidRequest("request body could not be read")
case buf.Len() == 0:
return nil, apperr.InvalidRequest("request body is empty")
}
return buf.Bytes(), nil
}
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
methodNotAllowed(w, r)
return
}
body := r.Body
defer func() { _, _ = io.CopyN(io.Discard, body, maxDrainBytes) }()
id := requestID(r)
chatID, ok := s.chatAccess(w, r)
if !ok {
return
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(query["filename"]) != 1 {
apperr.Write(w, id, apperr.InvalidRequest("exactly one filename query parameter is required"))
return
}
filename := query.Get("filename")
if !validFilename(filename) {
apperr.Write(w, id, apperr.InvalidRequest("filename must be 1..255 code points without path separators or control characters"))
return
}
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
if media != mimePNG && media != mimeJPEG {
apperr.Write(w, id, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
return
}
if r.ContentLength > s.maxUpload {
apperr.Write(w, id, apperr.PayloadTooLarge("attachment exceeds "+strconv.FormatInt(s.maxUpload, 10)+" bytes"))
return
}
if s.maxActiveJobs > 0 {
n, err := s.store.ActiveJobs(r.Context())
if err != nil {
s.storeError(w, r, err)
return
}
if n >= s.maxActiveJobs {
w.Header().Set("Retry-After", "5")
apperr.Write(w, id, apperr.Unavailable("attachment processing queue is full, retry later"))
return
}
}
if err := s.uploadSem.Acquire(r.Context(), 1); err != nil {
apperr.Write(w, id, apperr.Unavailable("too many concurrent uploads, retry later"))
return
}
defer s.uploadSem.Release(1)
data, aerr := readBody(w, r, s.maxUpload)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
info, aerr := s.inspectImage(data, media)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
sum := sha256.Sum256(data)
attID := uuid.NewString()
att := &model.Attachment{
ID: attID, ChatID: chatID, OwnerID: currentUser(r).ID, Filename: filename,
ContentType: info.contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]),
Status: "queued", Variants: []model.Variant{},
ObjectKey: "attachments/" + chatID + "/" + attID,
}
stagingKey := "staging/" + attID
if err := s.store.InsertPendingAttachment(r.Context(), att, info.width, info.height, stagingKey); err != nil {
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return
}
s.storeError(w, r, err)
return
}
putCtx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
err = s.objects.Put(putCtx, stagingKey, data, att.ContentType)
cancel()
if err != nil {
go s.compensateUpload(att, stagingKey, id)
s.objectError(w, r, err)
return
}
_, updated, err := s.store.QueueAttachment(r.Context(), att.ID, s.pipelineVersion, s.maxAttempts, s.maxActiveJobs)
if err != nil {
go s.compensateUpload(att, stagingKey, id)
if errors.Is(err, store.ErrQueueFull) {
w.Header().Set("Retry-After", "5")
apperr.Write(w, id, apperr.Unavailable("attachment processing queue is full, retry later"))
return
}
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.Unavailable("upload expired, retry"))
return
}
s.storeError(w, r, err)
return
}
if s.faultInjection && strings.HasPrefix(filename, "fault-api-after-commit") {
s.log.Warn("FAULT: exit after COMMIT before publish/response", "attachment_id", att.ID)
os.Exit(42)
}
if s.relay != nil {
s.relay.Kick()
}
att.UpdatedAt = model.NewTime(updated)
writeJSON(w, r, http.StatusAccepted, att)
}
func (s *Server) compensateUpload(att *model.Attachment, stagingKey, reqID string) {
dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second)
defer dcancel()
deleted, err := s.store.DeletePendingAttachment(dctx, att.ID)
if err != nil || !deleted {
if err != nil {
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
"attachment_id", att.ID, "stage", "delete_row")
}
return
}
octx, ocancel := context.WithTimeout(context.Background(), 3*time.Second)
defer ocancel()
if err := s.objects.Remove(octx, stagingKey); err != nil {
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
"attachment_id", att.ID, "stage", "remove_object")
}
}
func (s *Server) attachmentForRequest(w http.ResponseWriter, r *http.Request) (*model.Attachment, bool) {
id := requestID(r)
attID, err := parseUUID(r.PathValue("attachment_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("attachment_id must be a UUID"))
return nil, false
}
att, err := s.store.AttachmentForUser(r.Context(), attID, currentUser(r).ID)
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("attachment not found or not accessible"))
return nil, false
}
if err != nil {
s.storeError(w, r, err)
return nil, false
}
return att, true
}
func (s *Server) getAttachment(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
if att, ok := s.attachmentForRequest(w, r); ok {
writeJSON(w, r, http.StatusOK, att)
}
}
func (s *Server) downloadAttachment(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
id := requestID(r)
att, ok := s.attachmentForRequest(w, r)
if !ok {
return
}
variant := r.URL.Query().Get("variant")
switch variant {
case "", "original", "thumbnail", "crop", "optimized":
default:
apperr.Write(w, id, apperr.InvalidRequest("unknown variant"))
return
}
if att.Status != "ready" {
apperr.Write(w, id, apperr.Conflict("attachment is "+att.Status+", content is not available yet"))
return
}
key, contentType, etag := att.ObjectKey, att.ContentType, att.SHA256
if variant != "" && variant != "original" {
found := false
for _, v := range att.Variants {
if v.Name == variant {
key, contentType, etag, found = v.ObjectKey, v.ContentType, v.SHA256, true
}
}
if !found {
apperr.Write(w, id, apperr.NotFound("variant not found"))
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
defer cancel()
obj, size, err := s.objects.Get(ctx, key)
if objstore.IsNotFound(err) {
s.log.Error("attachment object missing in S3", "request_id", id, "attachment_id", att.ID)
apperr.Write(w, id, apperr.NotFound("attachment content is not available"))
return
}
if err != nil {
s.objectError(w, r, err)
return
}
defer obj.Close()
h := w.Header()
h.Set("Content-Type", contentType)
h.Set("Content-Length", strconv.FormatInt(size, 10))
h.Set("X-Content-Type-Options", "nosniff")
h.Set("Cache-Control", "private, no-store")
h.Set("Content-Disposition", "inline; filename*=UTF-8''"+url.PathEscape(att.Filename))
h.Set("ETag", `"`+etag+`"`)
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, obj); err != nil {
s.log.Warn("download interrupted", "request_id", id, "attachment_id", att.ID, "error", err.Error())
}
}
+431
View File
@@ -0,0 +1,431 @@
package api
import (
"context"
"encoding/base64"
"errors"
"net/http"
"strings"
"time"
"unicode/utf8"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/store"
)
func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) {
id := requestID(r)
if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) {
s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error())
apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable"))
return
}
s.log.Error("unexpected storage error", "request_id", id, "error", err.Error())
apperr.Write(w, id, apperr.Internal("internal error"))
}
func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) {
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
}
func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) {
if !s.ready.Load() {
apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down"))
return
}
ctx, cancel := context.WithTimeout(r.Context(), time.Second)
defer cancel()
if err := s.store.Ping(ctx); err != nil {
apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable"))
return
}
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
}
type createUserReq struct {
Username string `json:"username"`
DisplayName string `json:"display_name"`
Password string `json:"password"`
}
func validPassword(p string) bool {
n := utf8.RuneCountInString(p)
return utf8.ValidString(p) && n >= 12 && n <= 128
}
func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
methodNotAllowed(w, r)
return
}
id := requestID(r)
var req createUserReq
if aerr := decodeJSON(w, r, &req); aerr != nil {
apperr.Write(w, id, aerr)
return
}
if !usernameRe.MatchString(req.Username) {
apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$"))
return
}
if !validText(req.DisplayName, 100) {
apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points"))
return
}
if !validPassword(req.Password) {
apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points"))
return
}
hash, err := s.hasher.Hash(r.Context(), req.Password)
if err != nil {
s.storeError(w, r, err)
return
}
u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash)
if errors.Is(err, store.ErrUsernameTaken) {
apperr.Write(w, id, apperr.Conflict("username is already taken"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusCreated, u)
}
func (s *Server) me(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
writeJSON(w, r, http.StatusOK, currentUser(r))
}
const basicChallenge = `Basic realm="messenger", charset="UTF-8"`
func unauthorizedBasic(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", basicChallenge)
apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password"))
}
func parseBasic(r *http.Request) (string, string, bool) {
scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ")
if !ok || !strings.EqualFold(scheme, "Basic") {
return "", "", false
}
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value))
if err != nil || !utf8.Valid(raw) {
return "", "", false
}
user, pass, ok := strings.Cut(string(raw), ":")
if !ok || !usernameRe.MatchString(user) || !validPassword(pass) {
return "", "", false
}
return user, pass, true
}
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
methodNotAllowed(w, r)
return
}
username, password, ok := parseBasic(r)
if !ok {
unauthorizedBasic(w, r)
return
}
user, hash, err := s.store.Credentials(r.Context(), username)
if err != nil && !errors.Is(err, store.ErrNotFound) {
s.storeError(w, r, err)
return
}
valid, err := s.hasher.Verify(r.Context(), password, hash)
if err != nil {
s.storeError(w, r, err)
return
}
if !valid {
unauthorizedBasic(w, r)
return
}
token, expires, err := s.sessions.Create(r.Context(), user)
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusCreated, model.Session{
Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user,
})
}
func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
methodNotAllowed(w, r)
return
}
err := s.sessions.Revoke(r.Context(), currentToken(r))
if errors.Is(err, auth.ErrInvalidSession) {
unauthorizedBearer(w, r)
return
}
if err != nil {
s.storeError(w, r, err)
return
}
w.Header().Set("X-Request-Id", requestID(r))
w.WriteHeader(http.StatusNoContent)
}
type createChatReq struct {
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
}
func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
s.createChat(w, r)
case http.MethodGet:
s.listChats(w, r)
default:
methodNotAllowed(w, r)
}
}
func (s *Server) createChat(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
var req createChatReq
if aerr := decodeJSON(w, r, &req); aerr != nil {
apperr.Write(w, id, aerr)
return
}
if !validText(req.Title, 100) {
apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points"))
return
}
if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 {
apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items"))
return
}
seen := make(map[string]bool, len(req.MemberIDs))
for _, raw := range req.MemberIDs {
uid, err := parseUUID(raw)
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings"))
return
}
if seen[uid] {
apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id"))
return
}
seen[uid] = true
}
caller := currentUser(r)
if !seen[caller.ID] {
apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids"))
return
}
chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs)
if errors.Is(err, store.ErrUnknownMember) {
apperr.Write(w, id, apperr.InvalidRequest("unknown member_id"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusCreated, chat)
}
func (s *Server) listChats(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
caller := currentUser(r)
limit, cur, aerr := pagingParams(r, caller.ID)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur)
if err != nil {
s.storeError(w, r, err)
return
}
page := model.Page[*model.Chat]{Items: items}
if more && len(items) > 0 {
last := items[len(items)-1]
encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID})
page.NextCursor = &encoded
}
writeJSON(w, r, http.StatusOK, page)
}
func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
id := requestID(r)
chatID, err := parseUUID(r.PathValue("chat_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
return
}
chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID)
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusOK, chat)
}
type createMessageReq struct {
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}
func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
s.createMessage(w, r)
case http.MethodGet:
s.listMessages(w, r)
default:
methodNotAllowed(w, r)
}
}
func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) {
id := requestID(r)
chatID, err := parseUUID(r.PathValue("chat_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
return "", false
}
ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID)
if err != nil {
s.storeError(w, r, err)
return "", false
}
if !ok {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return "", false
}
return chatID, true
}
func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
chatID, err := parseUUID(r.PathValue("chat_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
return
}
reject := func(aerr *apperr.AppError) {
if _, ok := s.chatAccess(w, r); !ok {
return
}
apperr.Write(w, id, aerr)
}
key := r.Header.Get("Idempotency-Key")
if !idempotencyKeyRe.MatchString(key) {
reject(apperr.InvalidRequest("Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}"))
return
}
var req createMessageReq
if aerr := decodeJSON(w, r, &req); aerr != nil {
reject(aerr)
return
}
if req.AttachmentIDs == nil {
req.AttachmentIDs = []string{}
}
if len(req.AttachmentIDs) > 10 {
reject(apperr.InvalidRequest("attachment_ids must contain at most 10 items"))
return
}
seen := make(map[string]bool, len(req.AttachmentIDs))
for _, raw := range req.AttachmentIDs {
aid, err := parseUUID(raw)
if err != nil {
reject(apperr.InvalidRequest("attachment_ids must be UUID strings"))
return
}
if seen[aid] {
reject(apperr.InvalidRequest("duplicate attachment_id"))
return
}
seen[aid] = true
}
if len(req.AttachmentIDs) == 0 && !validText(req.Text, 4000) {
reject(apperr.InvalidRequest("text must be 1..4000 non-space code points when there are no attachments"))
return
}
if !utf8.ValidString(req.Text) || runeLen(req.Text) > 4000 {
reject(apperr.InvalidRequest("text must be at most 4000 code points"))
return
}
msg, replayed, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text, req.AttachmentIDs,
key, messageFingerprint(req), s.idempotencyTTL)
if errors.Is(err, store.ErrIdempotencyConflict) {
apperr.Write(w, id, apperr.Conflict("Idempotency-Key was already used with a different request"))
return
}
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return
}
if errors.Is(err, store.ErrAttachmentFailed) {
apperr.Write(w, id, apperr.Conflict("a failed attachment cannot be attached"))
return
}
if errors.Is(err, store.ErrAttachmentNotFound) {
apperr.Write(w, id, apperr.NotFound("attachment not found in this chat"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
if replayed {
w.Header().Set("Idempotent-Replayed", "true")
}
writeJSON(w, r, http.StatusCreated, msg)
}
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
chatID, ok := s.chatAccess(w, r)
if !ok {
return
}
limit, cur, aerr := pagingParams(r, chatID)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur)
if err != nil {
s.storeError(w, r, err)
return
}
page := model.Page[*model.Message]{Items: items}
if more && len(items) > 0 {
last := items[len(items)-1]
encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID})
page.NextCursor = &encoded
}
writeJSON(w, r, http.StatusOK, page)
}
func (s *Server) notFound(w http.ResponseWriter, r *http.Request) {
apperr.Write(w, requestID(r), apperr.NotFound("route not found"))
}
+110
View File
@@ -0,0 +1,110 @@
package api
import (
"crypto/sha256"
"encoding/json"
"errors"
"io"
"mime"
"net/http"
"regexp"
"strconv"
"strings"
"unicode/utf8"
"github.com/google/uuid"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/store"
)
const maxBodyBytes = 1 << 20
var (
usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
)
func messageFingerprint(req createMessageReq) []byte {
b, _ := json.Marshal(req)
sum := sha256.Sum256(b)
return sum[:]
}
func parseUUID(s string) (string, error) {
u, err := uuid.Parse(s)
if err != nil || u.String() != s {
return "", errors.New("invalid uuid")
}
return u.String(), nil
}
func runeLen(s string) int { return utf8.RuneCountInString(s) }
func validText(s string, max int) bool {
n := runeLen(s)
return n >= 1 && n <= max && strings.TrimSpace(s) != ""
}
func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError {
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
if media != "application/json" {
return apperr.UnsupportedMedia("Content-Type must be application/json")
}
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
dec := json.NewDecoder(r.Body)
dec.DisallowUnknownFields()
if err := dec.Decode(dst); err != nil {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
return apperr.PayloadTooLarge("request body too large")
}
return apperr.InvalidRequest("malformed JSON body")
}
var extra json.RawMessage
if err := dec.Decode(&extra); err != io.EOF {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
return apperr.PayloadTooLarge("request body too large")
}
return apperr.InvalidRequest("malformed JSON body")
}
return nil
}
func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) {
limit := 50
if v := r.URL.Query().Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 100 {
return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100")
}
limit = n
}
var cur *store.Cursor
if v := r.URL.Query().Get("cursor"); v != "" {
c, err := store.DecodeCursor(v)
if err == nil {
_, err = parseUUID(c.ID)
}
if err != nil {
return 0, nil, apperr.InvalidRequest("invalid cursor")
}
if c.Scope != scope {
return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource")
}
cur = &c
}
return limit, cur, nil
}
func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("X-Request-Id", requestID(r))
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(payload)
}
func methodNotAllowed(w http.ResponseWriter, r *http.Request) {
apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed"))
}
+149
View File
@@ -0,0 +1,149 @@
package api
import (
"context"
"errors"
"log/slog"
"net/http"
"strings"
"time"
"github.com/google/uuid"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/model"
)
type ctxKey int
const (
ctxRequestID ctxKey = iota
ctxUser
ctxToken
)
func requestIDMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
id := r.Header.Get("X-Request-Id")
if id == "" {
id = uuid.NewString()
}
w.Header().Set("X-Request-Id", id)
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id)))
})
}
func instanceIDMiddleware(instanceID string, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", instanceID)
next.ServeHTTP(w, r)
})
}
func requestID(r *http.Request) string {
if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" {
return id
}
return "unknown"
}
type statusWriter struct {
http.ResponseWriter
status int
}
func (s *statusWriter) WriteHeader(code int) {
s.status = code
s.ResponseWriter.WriteHeader(code)
}
func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(sw, r)
log.Info("http",
"method", r.Method,
"path", r.URL.Path,
"status", sw.status,
"dur_ms", time.Since(start).Milliseconds(),
"request_id", requestID(r),
)
})
}
func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if rec := recover(); rec != nil {
log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r))
apperr.Write(w, requestID(r), apperr.Internal("internal error"))
}
}()
next.ServeHTTP(w, r)
})
}
func timeoutMiddleware(d, binary time.Duration, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
limit := d
if strings.Contains(r.URL.Path, "/attachments") {
limit = binary
}
ctx, cancel := context.WithTimeout(r.Context(), limit)
defer cancel()
next.ServeHTTP(w, r.WithContext(ctx))
})
}
const bearerChallenge = `Bearer realm="messenger"`
func unauthorizedBearer(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", bearerChallenge)
apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required"))
}
func bearerToken(r *http.Request) (string, bool) {
scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ")
if !ok || !strings.EqualFold(scheme, "Bearer") {
return "", false
}
token = strings.TrimSpace(token)
if token == "" || len(token) > 256 {
return "", false
}
return token, true
}
func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
token, ok := bearerToken(r)
if !ok {
unauthorizedBearer(w, r)
return
}
user, err := s.sessions.Resolve(r.Context(), token)
if errors.Is(err, auth.ErrInvalidSession) {
unauthorizedBearer(w, r)
return
}
if err != nil {
s.storeError(w, r, err)
return
}
ctx := context.WithValue(r.Context(), ctxUser, user)
ctx = context.WithValue(ctx, ctxToken, token)
next.ServeHTTP(w, r.WithContext(ctx))
}
}
func currentUser(r *http.Request) *model.User {
return r.Context().Value(ctxUser).(*model.User)
}
func currentToken(r *http.Request) string {
return r.Context().Value(ctxToken).(string)
}
+92
View File
@@ -0,0 +1,92 @@
package api
import (
"log/slog"
"net/http"
"sync/atomic"
"time"
"golang.org/x/sync/semaphore"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/relay"
"mtusi-messenger/internal/store"
)
type Server struct {
store *store.Store
objects *objstore.Store
sessions *auth.Sessions
hasher *auth.Hasher
log *slog.Logger
ready *atomic.Bool
timeout time.Duration
instanceID string
idempotencyTTL time.Duration
uploadTimeout time.Duration
s3Timeout time.Duration
maxUpload int64
maxPixels int64
uploadSem *semaphore.Weighted
relay *relay.Relay
maxActiveJobs int
pipelineVersion int
maxAttempts int
faultInjection bool
}
type Options struct {
InstanceID string
IdempotencyTTL time.Duration
MaxUploadBytes int64
MaxImagePixels int64
UploadConcurrency int64
UploadTimeout time.Duration
S3Timeout time.Duration
Relay *relay.Relay
MaxActiveJobs int
PipelineVersion int
MaxAttempts int
FaultInjection bool
}
func NewServer(st *store.Store, objects *objstore.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool, opt Options) *Server {
return &Server{store: st, objects: objects, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second,
instanceID: opt.InstanceID, idempotencyTTL: opt.IdempotencyTTL,
uploadTimeout: opt.UploadTimeout, s3Timeout: opt.S3Timeout,
maxUpload: opt.MaxUploadBytes, maxPixels: opt.MaxImagePixels,
uploadSem: semaphore.NewWeighted(opt.UploadConcurrency),
relay: opt.Relay, maxActiveJobs: opt.MaxActiveJobs, pipelineVersion: opt.PipelineVersion,
maxAttempts: opt.MaxAttempts, faultInjection: opt.FaultInjection}
}
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/health/live", s.healthLive)
mux.HandleFunc("/health/ready", s.healthReady)
mux.HandleFunc("/api/v1/users", s.usersRoute)
mux.HandleFunc("/api/v1/auth/sessions", s.createSession)
mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession))
mux.HandleFunc("/api/v1/users/me", s.auth(s.me))
mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute))
mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute))
mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute))
mux.HandleFunc("/api/v1/chats/{chat_id}/attachments", s.auth(s.uploadAttachment))
mux.HandleFunc("/api/v1/attachments/{attachment_id}", s.auth(s.getAttachment))
mux.HandleFunc("/api/v1/attachments/{attachment_id}/content", s.auth(s.downloadAttachment))
mux.HandleFunc("/", s.notFound)
var h http.Handler = mux
h = timeoutMiddleware(s.timeout, s.uploadTimeout, h)
h = recoverMiddleware(s.log, h)
h = loggingMiddleware(s.log, h)
h = requestIDMiddleware(h)
h = instanceIDMiddleware(s.instanceID, h)
return h
}
+64
View File
@@ -0,0 +1,64 @@
package apperr
import (
"encoding/json"
"net/http"
)
type Code string
const (
CodeInvalidRequest Code = "invalid_request"
CodeUnauthorized Code = "unauthorized"
CodeNotFound Code = "not_found"
CodeConflict Code = "conflict"
CodePayloadTooLarge Code = "payload_too_large"
CodeUnsupportedMedia Code = "unsupported_media_type"
CodeRateLimited Code = "rate_limited"
CodeUnavailable Code = "unavailable"
CodeInternal Code = "internal_error"
)
type Body struct {
Code Code `json:"code"`
Message string `json:"message"`
RequestID string `json:"request_id"`
}
type Envelope struct {
Error Body `json:"error"`
}
type AppError struct {
Status int
Code Code
Msg string
}
func (e *AppError) Error() string { return e.Msg }
func New(status int, code Code, msg string) *AppError {
return &AppError{Status: status, Code: code, Msg: msg}
}
func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) }
func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) }
func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) }
func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) }
func PayloadTooLarge(msg string) *AppError {
return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg)
}
func UnsupportedMedia(msg string) *AppError {
return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg)
}
func Unavailable(msg string) *AppError {
return New(http.StatusServiceUnavailable, CodeUnavailable, msg)
}
func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) }
func Write(w http.ResponseWriter, requestID string, err *AppError) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("X-Request-Id", requestID)
w.WriteHeader(err.Status)
_ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}})
}
+100
View File
@@ -0,0 +1,100 @@
package auth
import (
"context"
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"errors"
"fmt"
"strings"
"golang.org/x/crypto/argon2"
)
const (
argonMemoryKiB = 19 * 1024
argonTime = 2
argonThreads = 1
argonKeyLen = 32
argonSaltLen = 16
)
var ErrBadHash = errors.New("malformed password hash")
type Hasher struct {
slots chan struct{}
dummy string
}
func NewHasher(concurrency int) *Hasher {
h := &Hasher{slots: make(chan struct{}, concurrency)}
salt := make([]byte, argonSaltLen)
_, _ = rand.Read(salt)
h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen))
return h
}
func (h *Hasher) acquire(ctx context.Context) error {
select {
case h.slots <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (h *Hasher) release() { <-h.slots }
func encode(salt, key []byte) string {
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads,
base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key))
}
func (h *Hasher) Hash(ctx context.Context, password string) (string, error) {
if err := h.acquire(ctx); err != nil {
return "", err
}
defer h.release()
salt := make([]byte, argonSaltLen)
if _, err := rand.Read(salt); err != nil {
return "", err
}
return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil
}
func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) {
missing := encoded == ""
if missing {
encoded = h.dummy
}
parts := strings.Split(encoded, "$")
if len(parts) != 6 || parts[1] != "argon2id" {
return false, ErrBadHash
}
var version int
var mem uint32
var t uint32
var p uint8
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version {
return false, ErrBadHash
}
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil {
return false, ErrBadHash
}
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
if err != nil {
return false, ErrBadHash
}
want, err := base64.RawStdEncoding.DecodeString(parts[5])
if err != nil {
return false, ErrBadHash
}
if err := h.acquire(ctx); err != nil {
return false, err
}
got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want)))
h.release()
return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil
}
+137
View File
@@ -0,0 +1,137 @@
package auth
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"errors"
"log/slog"
"time"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/store"
)
var (
ErrInvalidSession = errors.New("invalid session")
ErrCacheUnavailable = errors.New("session cache unavailable")
)
type SessionCache interface {
Get(ctx context.Context, key string) (*CachedSession, error)
Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error
Del(ctx context.Context, key string) error
}
type CachedSession struct {
User model.User `json:"user"`
ExpiresAt time.Time `json:"exp"`
}
type Sessions struct {
store *store.Store
cache SessionCache
ttl time.Duration
cacheMaxTTL time.Duration
log *slog.Logger
}
func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions {
return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log}
}
func NewToken() (string, []byte, error) {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return "", nil, err
}
token := base64.RawURLEncoding.EncodeToString(raw)
return token, HashToken(token), nil
}
func HashToken(token string) []byte {
sum := sha256.Sum256([]byte(token))
return sum[:]
}
func cacheKey(hash []byte) string {
return "sess:" + base64.RawURLEncoding.EncodeToString(hash)
}
func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) {
token, hash, err := NewToken()
if err != nil {
return "", time.Time{}, err
}
expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl)
if err != nil {
return "", time.Time{}, err
}
s.cacheSet(ctx, hash, user, expires)
return token, expires, nil
}
func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) {
hash := HashToken(token)
if s.cache != nil {
cs, err := s.cache.Get(ctx, cacheKey(hash))
if err == nil && cs != nil {
if time.Now().Before(cs.ExpiresAt) {
u := cs.User
return &u, nil
}
} else if err != nil {
s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error())
}
}
user, expires, err := s.store.SessionUser(ctx, hash)
if errors.Is(err, store.ErrNotFound) {
return nil, ErrInvalidSession
}
if err != nil {
return nil, err
}
s.cacheSet(ctx, hash, user, expires)
return user, nil
}
func (s *Sessions) Revoke(ctx context.Context, token string) error {
hash := HashToken(token)
if s.cache != nil {
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
return ErrCacheUnavailable
}
}
err := s.store.RevokeSession(ctx, hash)
if errors.Is(err, store.ErrNotFound) {
return ErrInvalidSession
}
if err != nil {
return err
}
if s.cache != nil {
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
s.log.Warn("session cache delete after revoke failed", "error", err.Error())
}
}
return nil
}
func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) {
if s.cache == nil {
return
}
ttl := time.Until(expires)
if ttl > s.cacheMaxTTL {
ttl = s.cacheMaxTTL
}
if ttl <= 0 {
return
}
if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil {
s.log.Debug("session cache set failed", "error", err.Error())
}
}
+125
View File
@@ -0,0 +1,125 @@
package broker
import (
"context"
"errors"
"fmt"
"sync"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
const (
Exchange = "attachments"
RoutingKey = "attachment.process.v1"
Queue = "attachment.process.v1"
DLX = "attachments.dlx"
DLQ = "attachment.process.dlq"
)
func Declare(ch *amqp.Channel) error {
if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil {
return err
}
if err := ch.ExchangeDeclare(DLX, "fanout", true, false, false, false, nil); err != nil {
return err
}
if _, err := ch.QueueDeclare(DLQ, true, false, false, false, amqp.Table{"x-queue-type": "quorum"}); err != nil {
return err
}
if err := ch.QueueBind(DLQ, "", DLX, false, nil); err != nil {
return err
}
if _, err := ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{
"x-queue-type": "quorum",
"x-dead-letter-exchange": DLX,
"x-delivery-limit": int32(20),
}); err != nil {
return err
}
return ch.QueueBind(Queue, RoutingKey, Exchange, false, nil)
}
func Dial(url string) (*amqp.Connection, error) {
return amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Locale: "en_US",
Properties: amqp.Table{"connection_name": "messenger"}})
}
type Publisher struct {
url string
mu sync.Mutex
conn *amqp.Connection
ch *amqp.Channel
}
func NewPublisher(url string) *Publisher { return &Publisher{url: url} }
func (p *Publisher) channel() (*amqp.Channel, error) {
if p.ch != nil && !p.ch.IsClosed() && p.conn != nil && !p.conn.IsClosed() {
return p.ch, nil
}
p.closeLocked()
conn, err := Dial(p.url)
if err != nil {
return nil, err
}
ch, err := conn.Channel()
if err != nil {
conn.Close()
return nil, err
}
if err := ch.Confirm(false); err != nil {
conn.Close()
return nil, err
}
if err := Declare(ch); err != nil {
conn.Close()
return nil, err
}
p.conn, p.ch = conn, ch
return ch, nil
}
func (p *Publisher) closeLocked() {
if p.conn != nil {
_ = p.conn.Close()
}
p.conn, p.ch = nil, nil
}
func (p *Publisher) Publish(ctx context.Context, messageID string, body []byte) error {
p.mu.Lock()
defer p.mu.Unlock()
ch, err := p.channel()
if err != nil {
return fmt.Errorf("broker connect: %w", err)
}
conf, err := ch.PublishWithDeferredConfirmWithContext(ctx, Exchange, RoutingKey, false, false, amqp.Publishing{
ContentType: "application/json",
DeliveryMode: amqp.Persistent,
MessageId: messageID,
Type: "attachment.process.v1",
Timestamp: time.Now().UTC(),
Body: body,
})
if err != nil {
p.closeLocked()
return fmt.Errorf("broker publish: %w", err)
}
ok, err := conf.WaitContext(ctx)
if err != nil {
p.closeLocked()
return fmt.Errorf("broker confirm: %w", err)
}
if !ok {
return errors.New("broker nacked publish")
}
return nil
}
func (p *Publisher) Close() {
p.mu.Lock()
defer p.mu.Unlock()
p.closeLocked()
}
+80
View File
@@ -0,0 +1,80 @@
package cache
import (
"context"
"encoding/json"
"errors"
"sync/atomic"
"time"
"github.com/redis/go-redis/v9"
"mtusi-messenger/internal/auth"
)
var ErrOpen = errors.New("cache circuit open")
type Valkey struct {
rdb *redis.Client
openUntil atomic.Int64
cooldown time.Duration
}
func NewValkey(url string) (*Valkey, error) {
opt, err := redis.ParseURL(url)
if err != nil {
return nil, err
}
opt.DialTimeout = 500 * time.Millisecond
opt.ReadTimeout = 300 * time.Millisecond
opt.WriteTimeout = 300 * time.Millisecond
opt.PoolSize = 10
opt.MaxRetries = 0
return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil
}
func (v *Valkey) Close() error { return v.rdb.Close() }
func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) }
func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() }
func (v *Valkey) guard(err error) error {
if err != nil && !errors.Is(err, redis.Nil) {
v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano())
}
return err
}
func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) {
if v.open() {
return nil, ErrOpen
}
raw, err := v.rdb.Get(ctx, key).Bytes()
if errors.Is(err, redis.Nil) {
return nil, nil
}
if err := v.guard(err); err != nil {
return nil, err
}
var cs auth.CachedSession
if err := json.Unmarshal(raw, &cs); err != nil {
return nil, nil
}
return &cs, nil
}
func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error {
if v.open() {
return ErrOpen
}
raw, err := json.Marshal(s)
if err != nil {
return err
}
return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err())
}
func (v *Valkey) Del(ctx context.Context, key string) error {
return v.guard(v.rdb.Del(ctx, key).Err())
}
+122
View File
@@ -0,0 +1,122 @@
package db
import (
"context"
"embed"
"fmt"
"io/fs"
"log/slog"
"sort"
"strconv"
"strings"
"github.com/jackc/pgx/v5/pgxpool"
)
//go:embed migrations/*.sql
var migrationFS embed.FS
const advisoryLockID = 20260202
type migration struct {
version int
name string
sql string
}
func loadMigrations() ([]migration, error) {
entries, err := fs.ReadDir(migrationFS, "migrations")
if err != nil {
return nil, err
}
var out []migration
for _, e := range entries {
name := e.Name()
prefix, _, ok := strings.Cut(name, "_")
if !ok || !strings.HasSuffix(name, ".sql") {
return nil, fmt.Errorf("bad migration file name %q", name)
}
v, err := strconv.Atoi(prefix)
if err != nil {
return nil, fmt.Errorf("bad migration version in %q", name)
}
body, err := migrationFS.ReadFile("migrations/" + name)
if err != nil {
return nil, err
}
out = append(out, migration{version: v, name: name, sql: string(body)})
}
sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
for i := 1; i < len(out); i++ {
if out[i].version == out[i-1].version {
return nil, fmt.Errorf("duplicate migration version %d", out[i].version)
}
}
return out, nil
}
func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error {
migs, err := loadMigrations()
if err != nil {
return err
}
conn, err := pool.Acquire(ctx)
if err != nil {
return err
}
defer conn.Release()
if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil {
return err
}
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID)
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
version integer PRIMARY KEY,
name text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now())`); err != nil {
return err
}
applied := map[int]bool{}
rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations")
if err != nil {
return err
}
for rows.Next() {
var v int
if err := rows.Scan(&v); err != nil {
rows.Close()
return err
}
applied[v] = true
}
rows.Close()
if err := rows.Err(); err != nil {
return err
}
for _, m := range migs {
if applied[m.version] {
continue
}
tx, err := conn.Begin(ctx)
if err != nil {
return err
}
if _, err := tx.Exec(ctx, m.sql); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("migration %s: %w", m.name, err)
}
if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil {
_ = tx.Rollback(ctx)
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
log.Info("migration applied", "version", m.version, "name", m.name)
}
log.Info("migrations up to date", "total", len(migs))
return nil
}
+51
View File
@@ -0,0 +1,51 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id),
position smallint NOT NULL,
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000),
CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
expires_at timestamptz NOT NULL,
revoked_at timestamptz,
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32),
CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at)
);
CREATE INDEX sessions_user_idx ON sessions (user_id);
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
@@ -0,0 +1,13 @@
CREATE TABLE message_idempotency (
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
idem_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, idem_key),
CONSTRAINT message_idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT message_idempotency_hash_len CHECK (octet_length(request_hash) = 32)
);
CREATE INDEX message_idempotency_created_idx ON message_idempotency (created_at);
@@ -0,0 +1,44 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL,
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
object_key text NOT NULL,
status text NOT NULL,
missing_at timestamptz,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
updated_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
CONSTRAINT attachments_chat_id_key UNIQUE (chat_id, id),
CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000),
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready'))
);
CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending';
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_key UNIQUE (message_id, position),
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_same_chat_fk FOREIGN KEY (chat_id, attachment_id)
REFERENCES attachments (chat_id, id)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+75
View File
@@ -0,0 +1,75 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
ALTER TABLE attachments ADD COLUMN staging_key text;
ALTER TABLE attachments ADD COLUMN error_code text;
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
ALTER TABLE attachments ADD CONSTRAINT attachments_error_code
CHECK ((status = 'failed') = (error_code IS NOT NULL));
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
name text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT attachment_variants_object_key_key UNIQUE (object_key),
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachment_variants_dims CHECK (width > 0 AND height > 0 AND size_bytes > 0)
);
CREATE TABLE attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
status text NOT NULL,
attempt integer NOT NULL DEFAULT 1,
max_attempts integer NOT NULL,
next_attempt_at timestamptz,
lease_owner text,
lease_until timestamptz,
error_code text,
last_error text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
started_at timestamptz,
finished_at timestamptz,
CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version),
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND max_attempts >= 1)
);
CREATE INDEX attachment_jobs_active_idx ON attachment_jobs (status, lease_until)
WHERE status IN ('queued', 'processing');
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (finished_at) WHERE status = 'failed';
CREATE TABLE job_outbox (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
attempt integer NOT NULL,
payload jsonb NOT NULL,
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
publish_attempts integer NOT NULL DEFAULT 0,
last_error text
);
CREATE INDEX job_outbox_unpublished_idx ON job_outbox (available_at, id) WHERE published_at IS NULL;
CREATE TABLE job_events (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
attempt integer NOT NULL,
event text NOT NULL,
error_code text,
detail text,
actor text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
+31
View File
@@ -0,0 +1,31 @@
package db
import (
"context"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
type PoolOptions struct {
MaxConns int32
}
func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) {
cfg, err := pgxpool.ParseConfig(url)
if err != nil {
return nil, err
}
cfg.MaxConns = opt.MaxConns
cfg.MinConns = 1
cfg.MaxConnLifetime = 30 * time.Minute
cfg.MaxConnIdleTime = 5 * time.Minute
cfg.HealthCheckPeriod = 15 * time.Second
cfg.ConnConfig.ConnectTimeout = 3 * time.Second
rp := cfg.ConnConfig.RuntimeParams
rp["application_name"] = "mtusi-messenger"
rp["statement_timeout"] = "5000"
rp["lock_timeout"] = "2000"
rp["idle_in_transaction_session_timeout"] = "10000"
return pgxpool.NewWithConfig(ctx, cfg)
}
+289
View File
@@ -0,0 +1,289 @@
package imaging
import (
"bytes"
"context"
"errors"
"fmt"
"image"
"image/color"
"image/jpeg"
"image/png"
"github.com/gen2brain/webp"
"golang.org/x/image/draw"
"golang.org/x/sync/semaphore"
)
const PipelineVersion = 1
const (
ThumbBox = 256
CropSide = 128
OptimizedBox = 1280
JPEGQuality = 85
WebPQuality = 80
WebPMethod = 4
)
type PermanentError struct{ Code, Detail string }
func (e *PermanentError) Error() string { return e.Code + ": " + e.Detail }
func permanent(code, format string, a ...any) error {
return &PermanentError{Code: code, Detail: fmt.Sprintf(format, a...)}
}
type Output struct {
Name string
ContentType string
Ext string
Data []byte
Width int
Height int
}
type Limits struct {
MaxBytes int64
MaxPixels int64
Budget *semaphore.Weighted
BudgetSize int64
}
func Fit(w, h, box int) (int, int) {
if w <= box && h <= box {
return w, h
}
if w >= h {
return box, max(1, h*box/w)
}
return max(1, w*box/h), box
}
func Cost(w, h int) int64 {
px := int64(w) * int64(h)
return px*4*2 + OptimizedBox*OptimizedBox*4
}
func Process(ctx context.Context, data []byte, contentType string, lim Limits) ([]Output, error) {
if int64(len(data)) > lim.MaxBytes {
return nil, permanent("image_too_large", "original exceeds %d bytes", lim.MaxBytes)
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
return nil, permanent("invalid_image", "image header is invalid")
}
if (format == "png") != (contentType == "image/png") || (format == "jpeg") != (contentType == "image/jpeg") {
return nil, permanent("invalid_image", "format does not match content type")
}
if int64(cfg.Width)*int64(cfg.Height) > lim.MaxPixels {
return nil, permanent("image_too_large", "image exceeds %d pixels", lim.MaxPixels)
}
cost := min(Cost(cfg.Width, cfg.Height), lim.BudgetSize)
if err := lim.Budget.Acquire(ctx, cost); err != nil {
return nil, err
}
type result struct {
out []Output
err error
}
done := make(chan result, 1)
go func() {
defer lim.Budget.Release(cost)
defer func() {
if p := recover(); p != nil {
done <- result{err: permanent("invalid_image", "decoder panic")}
}
}()
out, err := build(data, format)
done <- result{out, err}
}()
select {
case r := <-done:
return r.out, r.err
case <-ctx.Done():
return nil, ctx.Err()
}
}
func build(data []byte, format string) ([]Output, error) {
var src image.Image
var err error
orientation := 1
if format == "png" {
src, err = png.Decode(bytes.NewReader(data))
} else {
src, err = jpeg.Decode(bytes.NewReader(data))
orientation = jpegOrientation(data)
}
if err != nil {
return nil, permanent("invalid_image", "image data is corrupted")
}
img := flatten(orient(src, orientation))
b := img.Bounds()
w, h := b.Dx(), b.Dy()
var outs []Output
tw, th := Fit(w, h, ThumbBox)
thumb, err := encodeJPEG(resize(img, b, tw, th))
if err != nil {
return nil, err
}
outs = append(outs, Output{Name: "thumbnail", ContentType: "image/jpeg", Ext: "jpg", Data: thumb, Width: tw, Height: th})
side := min(w, h)
sq := image.Rect(b.Min.X+(w-side)/2, b.Min.Y+(h-side)/2, b.Min.X+(w-side)/2+side, b.Min.Y+(h-side)/2+side)
crop, err := encodeJPEG(resize(img, sq, CropSide, CropSide))
if err != nil {
return nil, err
}
outs = append(outs, Output{Name: "crop", ContentType: "image/jpeg", Ext: "jpg", Data: crop, Width: CropSide, Height: CropSide})
ow, oh := Fit(w, h, OptimizedBox)
var opt image.Image = img
if ow != w || oh != h {
opt = resize(img, b, ow, oh)
}
var buf bytes.Buffer
if err := webp.Encode(&buf, opt, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
return nil, fmt.Errorf("webp encode: %w", err)
}
outs = append(outs, Output{Name: "optimized", ContentType: "image/webp", Ext: "webp", Data: buf.Bytes(), Width: ow, Height: oh})
return outs, nil
}
func resize(src image.Image, sr image.Rectangle, w, h int) *image.RGBA {
dst := image.NewRGBA(image.Rect(0, 0, w, h))
draw.CatmullRom.Scale(dst, dst.Bounds(), src, sr, draw.Src, nil)
return dst
}
func encodeJPEG(img image.Image) ([]byte, error) {
var buf bytes.Buffer
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func flatten(src image.Image) image.Image {
if op, ok := src.(interface{ Opaque() bool }); ok && op.Opaque() {
return src
}
b := src.Bounds()
dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
draw.Draw(dst, dst.Bounds(), src, b.Min, draw.Over)
return dst
}
func orient(src image.Image, o int) image.Image {
if o < 2 || o > 8 {
return src
}
b := src.Bounds()
w, h := b.Dx(), b.Dy()
dw, dh := w, h
if o >= 5 {
dw, dh = h, w
}
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
var dx, dy int
switch o {
case 2:
dx, dy = w-1-x, y
case 3:
dx, dy = w-1-x, h-1-y
case 4:
dx, dy = x, h-1-y
case 5:
dx, dy = y, x
case 6:
dx, dy = h-1-y, x
case 7:
dx, dy = h-1-y, w-1-x
case 8:
dx, dy = y, w-1-x
}
dst.Set(dx, dy, src.At(b.Min.X+x, b.Min.Y+y))
}
}
return dst
}
var errNoExif = errors.New("no exif")
func jpegOrientation(data []byte) int {
o, err := parseOrientation(data)
if err != nil {
return 1
}
return o
}
func parseOrientation(d []byte) (int, error) {
if len(d) < 4 || d[0] != 0xff || d[1] != 0xd8 {
return 0, errNoExif
}
i := 2
for i+4 <= len(d) {
if d[i] != 0xff {
return 0, errNoExif
}
marker := d[i+1]
if marker == 0xda || marker == 0xd9 {
return 0, errNoExif
}
l := int(d[i+2])<<8 | int(d[i+3])
if l < 2 || i+2+l > len(d) {
return 0, errNoExif
}
seg := d[i+4 : i+2+l]
if marker == 0xe1 && len(seg) > 14 && string(seg[:6]) == "Exif\x00\x00" {
return tiffOrientation(seg[6:])
}
i += 2 + l
}
return 0, errNoExif
}
func tiffOrientation(t []byte) (int, error) {
if len(t) < 8 {
return 0, errNoExif
}
var u16 func([]byte) int
var u32 func([]byte) int
switch string(t[:2]) {
case "II":
u16 = func(b []byte) int { return int(b[0]) | int(b[1])<<8 }
u32 = func(b []byte) int { return int(b[0]) | int(b[1])<<8 | int(b[2])<<16 | int(b[3])<<24 }
case "MM":
u16 = func(b []byte) int { return int(b[1]) | int(b[0])<<8 }
u32 = func(b []byte) int { return int(b[3]) | int(b[2])<<8 | int(b[1])<<16 | int(b[0])<<24 }
default:
return 0, errNoExif
}
off := u32(t[4:8])
if off < 8 || off+2 > len(t) {
return 0, errNoExif
}
n := u16(t[off : off+2])
for k := 0; k < n; k++ {
e := off + 2 + k*12
if e+12 > len(t) {
break
}
if u16(t[e:e+2]) == 0x0112 {
v := u16(t[e+8 : e+10])
if v >= 1 && v <= 8 {
return v, nil
}
return 0, errNoExif
}
}
return 0, errNoExif
}
+106
View File
@@ -0,0 +1,106 @@
package imaging
import (
"bytes"
"context"
"image"
"image/color"
"image/jpeg"
"image/png"
"testing"
"golang.org/x/sync/semaphore"
)
func lim() Limits {
return Limits{MaxBytes: 10 << 20, MaxPixels: 40_000_000, Budget: semaphore.NewWeighted(256 << 20), BudgetSize: 256 << 20}
}
func TestFit(t *testing.T) {
cases := []struct{ w, h, box, ew, eh int }{
{320, 200, 256, 256, 160},
{200, 320, 256, 160, 256},
{100, 50, 256, 100, 50},
{5000, 1, 256, 256, 1},
{1, 5000, 1280, 1, 1280},
}
for _, c := range cases {
if w, h := Fit(c.w, c.h, c.box); w != c.ew || h != c.eh {
t.Errorf("Fit(%d,%d,%d)=%d,%d want %d,%d", c.w, c.h, c.box, w, h, c.ew, c.eh)
}
}
}
func TestProcessPNGWithAlpha(t *testing.T) {
src := image.NewNRGBA(image.Rect(0, 0, 320, 200))
var buf bytes.Buffer
_ = png.Encode(&buf, src)
outs, err := Process(context.Background(), buf.Bytes(), "image/png", lim())
if err != nil {
t.Fatal(err)
}
want := map[string][2]int{"thumbnail": {256, 160}, "crop": {128, 128}, "optimized": {320, 200}}
for _, o := range outs {
if d := want[o.Name]; d != [2]int{o.Width, o.Height} {
t.Errorf("%s: %dx%d want %v", o.Name, o.Width, o.Height, d)
}
if o.Name == "thumbnail" {
img, err := jpeg.Decode(bytes.NewReader(o.Data))
if err != nil {
t.Fatal(err)
}
if img.Bounds().Dx() != 256 || img.Bounds().Dy() != 160 {
t.Fatalf("thumbnail real size %v", img.Bounds())
}
r, g, b, _ := img.At(10, 10).RGBA()
if r>>8 < 245 || g>>8 < 245 || b>>8 < 245 {
t.Errorf("alpha not composited on white: %d %d %d", r>>8, g>>8, b>>8)
}
}
}
if len(outs) != 3 {
t.Fatalf("got %d variants", len(outs))
}
}
func TestExifOrientation(t *testing.T) {
src := image.NewRGBA(image.Rect(0, 0, 320, 200))
for y := 0; y < 200; y++ {
for x := 0; x < 320; x++ {
src.Set(x, y, color.RGBA{uint8(x), uint8(y), 0, 255})
}
}
var buf bytes.Buffer
_ = jpeg.Encode(&buf, src, nil)
data := buf.Bytes()
tiff := []byte{'M', 'M', 0, 42, 0, 0, 0, 8, 0, 1, 0x01, 0x12, 0, 3, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0}
seg := append([]byte("Exif\x00\x00"), tiff...)
app1 := append([]byte{0xff, 0xe1, byte((len(seg) + 2) >> 8), byte(len(seg) + 2)}, seg...)
withExif := append(append([]byte{0xff, 0xd8}, app1...), data[2:]...)
if o := jpegOrientation(withExif); o != 6 {
t.Fatalf("orientation %d", o)
}
outs, err := Process(context.Background(), withExif, "image/jpeg", lim())
if err != nil {
t.Fatal(err)
}
if outs[0].Width != 160 || outs[0].Height != 256 {
t.Fatalf("thumbnail %dx%d, want 160x256", outs[0].Width, outs[0].Height)
}
}
func TestRejects(t *testing.T) {
if _, err := Process(context.Background(), []byte("\x89PNG\r\n\x1a\nnot really"), "image/png", lim()); err == nil {
t.Fatal("corrupt PNG accepted")
} else if pe, ok := err.(*PermanentError); !ok || pe.Code != "invalid_image" {
t.Fatalf("unexpected error %v", err)
}
l := lim()
l.MaxPixels = 100
src := image.NewRGBA(image.Rect(0, 0, 20, 20))
var buf bytes.Buffer
_ = png.Encode(&buf, src)
if _, err := Process(context.Background(), buf.Bytes(), "image/png", l); err == nil {
t.Fatal("pixel limit not enforced")
}
}
+93
View File
@@ -0,0 +1,93 @@
package model
import "time"
type Time struct {
time.Time
}
func NewTime(t time.Time) Time { return Time{Time: t} }
func (t Time) MarshalJSON() ([]byte, error) {
return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil
}
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt Time `json:"created_at"`
}
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt Time `json:"created_at"`
}
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt Time `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
ObjectKey string `json:"-"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt Time `json:"created_at"`
UpdatedAt Time `json:"updated_at"`
ObjectKey string `json:"-"`
PipelineVersion int `json:"-"`
}
type JobMessage struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
}
const JobType = "attachment.process.v1"
type Page[T any] struct {
Items []T `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type Health struct {
Status string `json:"status"`
}
type Session struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
ExpiresAt Time `json:"expires_at"`
User User `json:"user"`
}
+162
View File
@@ -0,0 +1,162 @@
package objstore
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type Config struct {
Endpoint string
Region string
Bucket string
AccessKey string
SecretKey string
}
type Store struct {
c *minio.Client
bucket string
}
type Object struct {
Key string
Size int64
LastModified time.Time
}
func New(cfg Config) (*Store, error) {
u, err := url.Parse(cfg.Endpoint)
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
return nil, errors.New("S3_ENDPOINT must be http(s)://host:port")
}
if cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
return nil, errors.New("S3 bucket and credentials are required")
}
minio.MaxRetry = 1
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
MaxIdleConnsPerHost: 32,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
}
c, err := minio.New(u.Host, &minio.Options{
Transport: transport,
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: u.Scheme == "https",
Region: cfg.Region,
BucketLookup: minio.BucketLookupPath,
})
if err != nil {
return nil, err
}
return &Store{c: c, bucket: cfg.Bucket}, nil
}
func (s *Store) Bucket() string { return s.bucket }
func (s *Store) Ping(ctx context.Context) error {
ok, err := s.c.BucketExists(ctx, s.bucket)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("bucket %q does not exist", s.bucket)
}
return nil
}
func (s *Store) EnsureBucket(ctx context.Context, region string) (bool, error) {
ok, err := s.c.BucketExists(ctx, s.bucket)
if err != nil || ok {
return false, err
}
if err := s.c.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: region}); err != nil {
if code := minio.ToErrorResponse(err).Code; code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" {
return false, nil
}
return false, err
}
return true, nil
}
func (s *Store) Put(ctx context.Context, key string, data []byte, contentType string) error {
_, err := s.c.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: contentType, SendContentMd5: true, DisableMultipart: true})
return err
}
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) {
obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, 0, err
}
info, err := obj.Stat()
if err != nil {
obj.Close()
return nil, 0, err
}
return obj, info.Size, nil
}
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
info, err := s.c.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
return Object{}, err
}
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
}
func (s *Store) Remove(ctx context.Context, key string) error {
return s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
}
func (s *Store) List(ctx context.Context, prefix string, fn func(Object) error) error {
for info := range s.c.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
if info.Err != nil {
return info.Err
}
if err := fn(Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}); err != nil {
return err
}
}
return ctx.Err()
}
func IsNotFound(err error) bool {
if err == nil {
return false
}
switch minio.ToErrorResponse(err).Code {
case "NoSuchKey", "NotFound":
return true
}
return false
}
func (s *Store) ReadAll(ctx context.Context, key string, limit int64) ([]byte, error) {
obj, size, err := s.Get(ctx, key)
if err != nil {
return nil, err
}
defer obj.Close()
if size > limit {
return nil, fmt.Errorf("object larger than %d bytes", limit)
}
buf := bytes.NewBuffer(make([]byte, 0, size))
if _, err := io.Copy(buf, io.LimitReader(obj, limit+1)); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
+73
View File
@@ -0,0 +1,73 @@
package relay
import (
"context"
"log/slog"
"time"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/store"
)
type Relay struct {
st *store.Store
pub *broker.Publisher
log *slog.Logger
kick chan struct{}
every time.Duration
failed bool
}
func New(st *store.Store, pub *broker.Publisher, log *slog.Logger, every time.Duration) *Relay {
return &Relay{st: st, pub: pub, log: log, kick: make(chan struct{}, 1), every: every}
}
func (r *Relay) Kick() {
select {
case r.kick <- struct{}{}:
default:
}
}
func (r *Relay) Run(ctx context.Context) {
t := time.NewTicker(r.every)
defer t.Stop()
for {
r.drain(ctx)
select {
case <-ctx.Done():
return
case <-t.C:
case <-r.kick:
}
}
}
func (r *Relay) drain(ctx context.Context) {
for ctx.Err() == nil {
pctx, cancel := context.WithTimeout(ctx, 15*time.Second)
n, err := r.st.RelayOutbox(pctx, 50, func(row store.OutboxRow) error {
c, cc := context.WithTimeout(pctx, 5*time.Second)
defer cc()
return r.pub.Publish(c, row.JobID, row.Payload)
})
cancel()
if err != nil {
if !r.failed {
r.log.Warn("outbox relay pass failed", "error", err.Error())
}
r.failed = true
return
}
if r.failed {
r.log.Info("outbox relay recovered")
r.failed = false
}
if n > 0 {
r.log.Debug("outbox published", "count", n)
}
if n < 50 {
return
}
}
}
+88
View File
@@ -0,0 +1,88 @@
package store
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"mtusi-messenger/internal/model"
)
const attachmentColumns = `a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
a.object_key, a.status, a.error_code, a.pipeline_version, a.created_at, a.updated_at`
func scanAttachment(row pgx.Row) (*model.Attachment, error) {
a := &model.Attachment{Variants: []model.Variant{}}
var created, updated time.Time
err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
&a.ObjectKey, &a.Status, &a.ErrorCode, &a.PipelineVersion, &created, &updated)
if err != nil {
return nil, err
}
a.CreatedAt, a.UpdatedAt = utc(created), utc(updated)
return a, nil
}
func (s *Store) InsertPendingAttachment(ctx context.Context, a *model.Attachment, width, height int, stagingKey string) error {
var created time.Time
err := s.pool.QueryRow(ctx, `
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, object_key, staging_key, status)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'pending')
RETURNING created_at`,
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
width, height, a.ObjectKey, stagingKey).Scan(&created)
if pgCode(err) == "23503" {
return ErrNotFound
}
if err != nil {
return err
}
a.CreatedAt = utc(created)
return nil
}
func (s *Store) DeletePendingAttachment(ctx context.Context, id string) (bool, error) {
tag, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
func (s *Store) AttachmentForUser(ctx context.Context, id, userID string) (*model.Attachment, error) {
a, err := scanAttachment(s.pool.QueryRow(ctx, `
SELECT `+attachmentColumns+`
FROM attachments a
WHERE a.id = $1 AND a.status <> 'pending'
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2)`,
id, userID))
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
if a.Status != "ready" {
return a, nil
}
rows, err := s.pool.Query(ctx, `
SELECT name, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2
ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END`, a.ID, a.PipelineVersion)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var v model.Variant
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil {
return nil, err
}
a.Variants = append(a.Variants, v)
}
return a, rows.Err()
}
+33
View File
@@ -0,0 +1,33 @@
package store
import (
"encoding/base64"
"encoding/json"
"time"
)
type Cursor struct {
Scope string `json:"s"`
TS time.Time `json:"t"`
ID string `json:"i"`
}
func EncodeCursor(c Cursor) string {
b, _ := json.Marshal(c)
return base64.RawURLEncoding.EncodeToString(b)
}
func DecodeCursor(s string) (Cursor, error) {
var c Cursor
b, err := base64.RawURLEncoding.DecodeString(s)
if err != nil {
return c, ErrInvalidCursor
}
if err := json.Unmarshal(b, &c); err != nil {
return c, ErrInvalidCursor
}
if c.Scope == "" || c.ID == "" || c.TS.IsZero() {
return c, ErrInvalidCursor
}
return c, nil
}
+493
View File
@@ -0,0 +1,493 @@
package store
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"mtusi-messenger/internal/model"
)
var (
ErrJobNotFailed = errors.New("job is not in failed state")
ErrJobAlreadyOK = errors.New("job already completed")
ErrStagingGone = errors.New("staging original already removed by retention")
ErrQueueFull = errors.New("too much unfinished work")
)
func (s *Store) inTx(ctx context.Context, fn func(pgx.Tx) error) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return err
}
defer tx.Rollback(context.Background())
if err := fn(tx); err != nil {
return err
}
return tx.Commit(ctx)
}
func jobPayload(jobID, attID string, pv, attempt int, created time.Time) []byte {
b, _ := json.Marshal(model.JobMessage{
SchemaVersion: 1, JobID: jobID, Type: model.JobType, AttachmentID: attID,
PipelineVersion: pv, Attempt: attempt, CreatedAt: created.UTC().Format(time.RFC3339Nano),
})
return b
}
func addOutbox(ctx context.Context, tx pgx.Tx, jobID, attID string, pv, attempt int, availableAt *time.Time) error {
_, err := tx.Exec(ctx, `
INSERT INTO job_outbox (job_id, attempt, payload, available_at)
VALUES ($1, $2, $3, COALESCE($4, clock_timestamp()))`,
jobID, attempt, jobPayload(jobID, attID, pv, attempt, time.Now()), availableAt)
return err
}
func addEvent(ctx context.Context, tx pgx.Tx, jobID string, attempt int, event string, code, detail *string, actor string) error {
_, err := tx.Exec(ctx, `
INSERT INTO job_events (job_id, attempt, event, error_code, detail, actor)
VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''))`, jobID, attempt, event, code, detail, actor)
return err
}
func (s *Store) QueueAttachment(ctx context.Context, attID string, pipelineVersion, maxAttempts, maxActive int) (string, time.Time, error) {
jobID := uuid.NewString()
var updated time.Time
err := s.inTx(ctx, func(tx pgx.Tx) error {
if maxActive > 0 {
var active int
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(20260505)`); err != nil {
return err
}
if err := tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')`).Scan(&active); err != nil {
return err
}
if active >= maxActive {
return ErrQueueFull
}
}
err := tx.QueryRow(ctx, `
UPDATE attachments SET status = 'queued', pipeline_version = $2,
updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status = 'pending'
RETURNING updated_at`, attID, pipelineVersion).Scan(&updated)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts)
VALUES ($1, $2, $3, 'queued', 1, $4)`, jobID, attID, pipelineVersion, maxAttempts); err != nil {
return err
}
if err := addEvent(ctx, tx, jobID, 1, "queued", nil, nil, "api"); err != nil {
return err
}
return addOutbox(ctx, tx, jobID, attID, pipelineVersion, 1, nil)
})
return jobID, updated.UTC(), err
}
func (s *Store) ActiveJobs(ctx context.Context) (int, error) {
var n int
err := s.pool.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')`).Scan(&n)
return n, err
}
type OutboxRow struct {
ID int64
JobID string
Attempt int
Payload []byte
}
func (s *Store) RelayOutbox(ctx context.Context, limit int, publish func(OutboxRow) error) (int, error) {
published := 0
err := s.inTx(ctx, func(tx pgx.Tx) error {
rows, err := tx.Query(ctx, `
SELECT id, job_id, attempt, payload FROM job_outbox
WHERE published_at IS NULL AND available_at <= clock_timestamp()
ORDER BY available_at, id
LIMIT $1
FOR UPDATE SKIP LOCKED`, limit)
if err != nil {
return err
}
var batch []OutboxRow
for rows.Next() {
var r OutboxRow
if err := rows.Scan(&r.ID, &r.JobID, &r.Attempt, &r.Payload); err != nil {
rows.Close()
return err
}
batch = append(batch, r)
}
rows.Close()
if err := rows.Err(); err != nil {
return err
}
for _, r := range batch {
if perr := publish(r); perr != nil {
msg := perr.Error()
_, _ = tx.Exec(ctx, `UPDATE job_outbox SET publish_attempts = publish_attempts + 1, last_error = $2 WHERE id = $1`, r.ID, msg)
return nil
}
if _, err := tx.Exec(ctx, `
UPDATE job_outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1, last_error = NULL
WHERE id = $1`, r.ID); err != nil {
return err
}
published++
}
return nil
})
if err != nil {
return 0, err
}
return published, nil
}
func (s *Store) PurgeOutbox(ctx context.Context, olderThan time.Duration) (int64, error) {
tag, err := s.pool.Exec(ctx, `DELETE FROM job_outbox WHERE published_at < now() - make_interval(secs => $1)`, olderThan.Seconds())
return tag.RowsAffected(), err
}
type JobWork struct {
JobID string
AttachmentID string
ChatID string
Filename string
ContentType string
SizeBytes int64
SHA256 string
ObjectKey string
StagingKey string
PipelineVersion int
Attempt int
MaxAttempts int
}
type ClaimResult string
const (
Claimed ClaimResult = "claimed"
Done ClaimResult = "done"
Stale ClaimResult = "stale"
Busy ClaimResult = "busy"
NotFound ClaimResult = "notfound"
NotYet ClaimResult = "notyet"
)
func (s *Store) ClaimJob(ctx context.Context, jobID string, attempt int, owner string, lease time.Duration, takeover bool) (*JobWork, ClaimResult, error) {
var w JobWork
result := Claimed
err := s.inTx(ctx, func(tx pgx.Tx) error {
var status string
var curAttempt int
var next *time.Time
err := tx.QueryRow(ctx, `SELECT status, attempt, next_attempt_at FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID).
Scan(&status, &curAttempt, &next)
if errors.Is(err, pgx.ErrNoRows) {
result = NotFound
return nil
}
if err != nil {
return err
}
switch {
case status == "ready" || status == "failed":
result = Done
return nil
case curAttempt != attempt:
result = Stale
return nil
case status == "processing" && !takeover:
var expired bool
if err := tx.QueryRow(ctx, `SELECT lease_until < clock_timestamp() FROM attachment_jobs WHERE job_id = $1`, jobID).Scan(&expired); err != nil {
return err
}
if !expired {
result = Busy
return nil
}
case status == "queued" && next != nil && next.After(time.Now().Add(2*time.Second)):
result = NotYet
return nil
}
err = tx.QueryRow(ctx, `
UPDATE attachment_jobs
SET status = 'processing', lease_owner = $2,
lease_until = clock_timestamp() + make_interval(secs => $3),
started_at = clock_timestamp(), updated_at = clock_timestamp()
WHERE job_id = $1
RETURNING attachment_id, pipeline_version, attempt, max_attempts`, jobID, owner, lease.Seconds()).
Scan(&w.AttachmentID, &w.PipelineVersion, &w.Attempt, &w.MaxAttempts)
if err != nil {
return err
}
err = tx.QueryRow(ctx, `
UPDATE attachments SET status = 'processing', updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status IN ('queued', 'processing')
RETURNING chat_id, filename, content_type, size_bytes, sha256, object_key, COALESCE(staging_key, '')`, w.AttachmentID).
Scan(&w.ChatID, &w.Filename, &w.ContentType, &w.SizeBytes, &w.SHA256, &w.ObjectKey, &w.StagingKey)
if errors.Is(err, pgx.ErrNoRows) {
result = NotFound
return errors.New("rollback")
}
if err != nil {
return err
}
ev := "started"
if status == "processing" {
ev = "taken_over"
}
return addEvent(ctx, tx, jobID, attempt, ev, nil, nil, owner)
})
if result == NotFound {
return nil, NotFound, nil
}
if err != nil {
return nil, "", err
}
if result != Claimed {
return nil, result, nil
}
w.JobID = jobID
return &w, Claimed, nil
}
func (s *Store) ExtendLease(ctx context.Context, jobID string, attempt int, owner string, lease time.Duration) (bool, error) {
tag, err := s.pool.Exec(ctx, `
UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $4)
WHERE job_id = $1 AND attempt = $2 AND lease_owner = $3 AND status = 'processing'`,
jobID, attempt, owner, lease.Seconds())
return tag.RowsAffected() == 1, err
}
func (s *Store) CompleteJob(ctx context.Context, w *JobWork, variants []model.Variant) (bool, error) {
ok := false
err := s.inTx(ctx, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'ready', lease_owner = NULL, lease_until = NULL, error_code = NULL,
finished_at = clock_timestamp(), updated_at = clock_timestamp()
WHERE job_id = $1 AND attempt = $2 AND status = 'processing'`, w.JobID, w.Attempt)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return nil
}
for _, v := range variants {
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type,
size_bytes, width, height, sha256, object_key)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING`,
w.AttachmentID, w.PipelineVersion, v.Name, v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256, v.ObjectKey); err != nil {
return err
}
}
if _, err := tx.Exec(ctx, `
UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status = 'processing'`, w.AttachmentID); err != nil {
return err
}
ok = true
return addEvent(ctx, tx, w.JobID, w.Attempt, "ready", nil, nil, "")
})
return ok, err
}
func (s *Store) ClearStaging(ctx context.Context, attID string) error {
_, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1 AND status IN ('ready', 'failed')`, attID)
return err
}
func (s *Store) FailOrRetry(ctx context.Context, jobID string, attempt int, code, detail string, permanent bool, backoff time.Duration, actor string) (string, error) {
out := ""
err := s.inTx(ctx, func(tx pgx.Tx) error {
var attID string
var pv, maxAttempts int
err := tx.QueryRow(ctx, `
SELECT attachment_id, pipeline_version, max_attempts FROM attachment_jobs
WHERE job_id = $1 AND attempt = $2 AND status = 'processing' FOR UPDATE`, jobID, attempt).
Scan(&attID, &pv, &maxAttempts)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
return failOrRetryLocked(ctx, tx, jobID, attID, pv, attempt, maxAttempts, code, detail, permanent, backoff, actor, &out)
})
return out, err
}
func failOrRetryLocked(ctx context.Context, tx pgx.Tx, jobID, attID string, pv, attempt, maxAttempts int, code, detail string, permanent bool, backoff time.Duration, actor string, out *string) error {
if !permanent && attempt >= maxAttempts {
permanent = true
detail = code + ": " + detail
code = "retry_exhausted"
}
if permanent {
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs SET status = 'failed', error_code = $2, last_error = $3,
lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp()
WHERE job_id = $1`, jobID, code, detail); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE attachments SET status = 'failed', error_code = $2, updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status IN ('queued', 'processing')`, attID, code); err != nil {
return err
}
*out = "failed"
return addEvent(ctx, tx, jobID, attempt, "failed", &code, &detail, actor)
}
next := time.Now().Add(backoff)
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs SET status = 'queued', attempt = attempt + 1, next_attempt_at = $2,
error_code = $3, last_error = $4, lease_owner = NULL, lease_until = NULL, updated_at = clock_timestamp()
WHERE job_id = $1`, jobID, next, code, detail); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE attachments SET status = 'queued', updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status IN ('queued', 'processing')`, attID); err != nil {
return err
}
if err := addEvent(ctx, tx, jobID, attempt, "retry_scheduled", &code, &detail, actor); err != nil {
return err
}
*out = "queued"
return addOutbox(ctx, tx, jobID, attID, pv, attempt+1, &next)
}
func (s *Store) RecoverExpiredLeases(ctx context.Context, limit int) (int, error) {
n := 0
err := s.inTx(ctx, func(tx pgx.Tx) error {
rows, err := tx.Query(ctx, `
SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts FROM attachment_jobs
WHERE status = 'processing' AND lease_until < clock_timestamp()
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED`, limit)
if err != nil {
return err
}
type rec struct {
job, att string
pv, attempt, maxAt int
}
var list []rec
for rows.Next() {
var r rec
if err := rows.Scan(&r.job, &r.att, &r.pv, &r.attempt, &r.maxAt); err != nil {
rows.Close()
return err
}
list = append(list, r)
}
rows.Close()
for _, r := range list {
var out string
if err := failOrRetryLocked(ctx, tx, r.job, r.att, r.pv, r.attempt, r.maxAt, "lease_expired",
"worker lease expired", false, 0, "sweeper", &out); err != nil {
return err
}
n++
}
return nil
})
return n, err
}
func (s *Store) ReplayFailed(ctx context.Context, attID, actor string, extraAttempts int) (string, int, error) {
var jobID string
var attempt int
err := s.inTx(ctx, func(tx pgx.Tx) error {
var status string
var pv int
var staging *string
err := tx.QueryRow(ctx, `
SELECT j.job_id, j.status, j.attempt, j.pipeline_version, a.staging_key
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.attachment_id = $1 ORDER BY j.pipeline_version DESC LIMIT 1
FOR UPDATE`, attID).Scan(&jobID, &status, &attempt, &pv, &staging)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
switch status {
case "ready":
return ErrJobAlreadyOK
case "failed":
default:
return ErrJobNotFailed
}
if staging == nil {
return ErrStagingGone
}
attempt++
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs SET status = 'queued', attempt = $2, max_attempts = $3, next_attempt_at = NULL,
error_code = NULL, finished_at = NULL, updated_at = clock_timestamp()
WHERE job_id = $1`, jobID, attempt, attempt+extraAttempts-1); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status = 'failed'`, attID); err != nil {
return err
}
if err := addEvent(ctx, tx, jobID, attempt, "replay", nil, nil, actor); err != nil {
return err
}
return addOutbox(ctx, tx, jobID, attID, pv, attempt, nil)
})
return jobID, attempt, err
}
type StagingRef struct {
AttachmentID string
StagingKey string
Status string
}
func (s *Store) StaleStaging(ctx context.Context, pendingGrace, failedRetention time.Duration, limit int) ([]StagingRef, error) {
rows, err := s.pool.Query(ctx, `
SELECT id, staging_key, status FROM attachments
WHERE staging_key IS NOT NULL
AND ((status = 'pending' AND created_at < now() - make_interval(secs => $1))
OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
LIMIT $3`, pendingGrace.Seconds(), failedRetention.Seconds(), limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []StagingRef
for rows.Next() {
var r StagingRef
if err := rows.Scan(&r.AttachmentID, &r.StagingKey, &r.Status); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
func (s *Store) ForgetStaging(ctx context.Context, r StagingRef) error {
if r.Status == "pending" {
_, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, r.AttachmentID)
return err
}
_, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1 AND status = 'failed'`, r.AttachmentID)
return err
}
+388
View File
@@ -0,0 +1,388 @@
package store
import (
"bytes"
"context"
"errors"
"net"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"mtusi-messenger/internal/model"
)
var (
ErrNotFound = errors.New("not found")
ErrUsernameTaken = errors.New("username taken")
ErrUnknownMember = errors.New("unknown member")
ErrInvalidCursor = errors.New("invalid cursor")
)
type Store struct {
pool *pgxpool.Pool
}
func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
func IsUnavailable(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) {
return true
}
var connErr *pgconn.ConnectError
if errors.As(err, &connErr) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) {
return true
}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
switch {
case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03",
len(pgErr.Code) == 5 && pgErr.Code[:2] == "08":
return true
}
}
return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed)
}
func pgCode(err error) string {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
return pgErr.Code
}
return ""
}
func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) }
func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) {
u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName}
var created time.Time
err := s.pool.QueryRow(ctx,
`INSERT INTO users (id, username, display_name, password_hash)
VALUES ($1, $2, $3, $4) RETURNING created_at`,
u.ID, username, displayName, passwordHash).Scan(&created)
if pgCode(err) == "23505" {
return nil, ErrUsernameTaken
}
if err != nil {
return nil, err
}
u.CreatedAt = utc(created)
return u, nil
}
func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) {
u := &model.User{}
var created time.Time
var hash string
err := s.pool.QueryRow(ctx,
`SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`,
username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash)
if errors.Is(err, pgx.ErrNoRows) {
return nil, "", ErrNotFound
}
if err != nil {
return nil, "", err
}
u.CreatedAt = utc(created)
return u, hash, nil
}
func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) {
var expires time.Time
err := s.pool.QueryRow(ctx,
`INSERT INTO sessions (token_hash, user_id, expires_at)
VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3))
RETURNING expires_at`,
tokenHash, userID, ttl.Seconds()).Scan(&expires)
if err != nil {
return time.Time{}, err
}
return expires.UTC(), nil
}
func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) {
u := &model.User{}
var created, expires time.Time
err := s.pool.QueryRow(ctx,
`SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
FROM sessions s JOIN users u ON u.id = s.user_id
WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`,
tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires)
if errors.Is(err, pgx.ErrNoRows) {
return nil, time.Time{}, ErrNotFound
}
if err != nil {
return nil, time.Time{}, err
}
u.CreatedAt = utc(created)
return u, expires.UTC(), nil
}
func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error {
tag, err := s.pool.Exec(ctx,
`UPDATE sessions SET revoked_at = now()
WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return ErrNotFound
}
return nil
}
func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) {
tag, err := s.pool.Exec(ctx,
`DELETE FROM sessions
WHERE expires_at < now() - make_interval(secs => $1)
OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds())
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) {
c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)}
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var created time.Time
if err := tx.QueryRow(ctx,
`INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`,
c.ID, title, creatorID).Scan(&created); err != nil {
return err
}
c.CreatedAt = utc(created)
_, err := tx.Exec(ctx,
`INSERT INTO chat_members (chat_id, user_id, position)
SELECT $1, m.user_id, m.ord - 1
FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`,
c.ID, memberIDs)
return err
})
if pgCode(err) == "23503" {
return nil, ErrUnknownMember
}
if err != nil {
return nil, err
}
return c, nil
}
func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) {
var ok bool
err := s.pool.QueryRow(ctx,
`SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`,
chatID, userID).Scan(&ok)
return ok, err
}
func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) {
c := &model.Chat{}
var created time.Time
err := s.pool.QueryRow(ctx,
`SELECT c.id, c.title, c.created_at,
array_agg(m.user_id::text ORDER BY m.position)
FROM chats c JOIN chat_members m ON m.chat_id = c.id
WHERE c.id = $1
AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2)
GROUP BY c.id`,
chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
c.CreatedAt = utc(created)
return c, nil
}
func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) {
var ts *time.Time
var cid *string
if cur != nil {
ts, cid = &cur.TS, &cur.ID
}
rows, err := s.pool.Query(ctx,
`SELECT c.id, c.title, c.created_at,
(SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id)
FROM chat_members me JOIN chats c ON c.id = me.chat_id
WHERE me.user_id = $1
AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid))
ORDER BY c.created_at, c.id
LIMIT $4`,
userID, ts, cid, limit+1)
if err != nil {
return nil, false, err
}
defer rows.Close()
out := make([]*model.Chat, 0, limit+1)
for rows.Next() {
c := &model.Chat{}
var created time.Time
if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil {
return nil, false, err
}
c.CreatedAt = utc(created)
out = append(out, c)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
more := len(out) > limit
if more {
out = out[:limit]
}
return out, more, nil
}
var (
ErrIdempotencyConflict = errors.New("idempotency key reused with different body")
ErrAttachmentNotFound = errors.New("attachment not found in chat")
ErrAttachmentFailed = errors.New("attachment processing failed")
)
func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, key string, reqHash []byte, ttl time.Duration) (*model.Message, bool, error) {
for attempt := 0; attempt < 2; attempt++ {
newID := uuid.NewString()
var member, attachmentsOK bool
var failedCount int64
var created *time.Time
err := s.pool.QueryRow(ctx, `
WITH member AS (
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1
), att AS (
SELECT count(*) FILTER (WHERE status IN ('queued', 'processing', 'ready')) = cardinality($8::uuid[]) AS ok,
count(*) FILTER (WHERE status = 'failed') AS failed
FROM attachments
WHERE chat_id = $2 AND id = ANY($8::uuid[])
), k AS (
INSERT INTO message_idempotency (user_id, chat_id, idem_key, request_hash, message_id)
SELECT $1, $2, $3, $4, $5 FROM member, att WHERE att.ok
ON CONFLICT (user_id, chat_id, idem_key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = now()
WHERE message_idempotency.created_at < now() - make_interval(secs => $7)
RETURNING message_id
), m AS (
INSERT INTO messages (id, chat_id, sender_id, text)
SELECT $5, $2, $1, $6 FROM k
RETURNING created_at
), ma AS (
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
SELECT $5, $2, a.id, a.ord - 1
FROM unnest($8::uuid[]) WITH ORDINALITY AS a(id, ord)
WHERE EXISTS (SELECT 1 FROM m)
)
SELECT EXISTS (SELECT 1 FROM member), (SELECT ok FROM att), (SELECT failed FROM att), (SELECT created_at FROM m)`,
senderID, chatID, key, reqHash, newID, text, ttl.Seconds(), attachmentIDs).Scan(&member, &attachmentsOK, &failedCount, &created)
if pgCode(err) == "23503" {
return nil, false, ErrAttachmentNotFound
}
if err != nil {
return nil, false, err
}
if !member {
return nil, false, ErrNotFound
}
if !attachmentsOK {
if failedCount > 0 {
return nil, false, ErrAttachmentFailed
}
return nil, false, ErrAttachmentNotFound
}
if created != nil {
return &model.Message{ID: newID, ChatID: chatID, SenderID: senderID, Text: text,
CreatedAt: utc(*created), AttachmentIDs: attachmentIDs}, false, nil
}
var stored []byte
var at time.Time
m := &model.Message{ChatID: chatID}
err = s.pool.QueryRow(ctx, `
SELECT i.request_hash, m.id, m.sender_id, m.text, m.created_at,
`+attachmentIDsSQL+`
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
WHERE i.user_id = $1 AND i.chat_id = $2 AND i.idem_key = $3
AND i.created_at >= now() - make_interval(secs => $4)`,
senderID, chatID, key, ttl.Seconds()).Scan(&stored, &m.ID, &m.SenderID, &m.Text, &at, &m.AttachmentIDs)
if errors.Is(err, pgx.ErrNoRows) {
continue
}
if err != nil {
return nil, false, err
}
if !bytes.Equal(stored, reqHash) {
return nil, false, ErrIdempotencyConflict
}
m.CreatedAt = utc(at)
return m, true, nil
}
return nil, false, errors.New("idempotency record kept changing")
}
const attachmentIDsSQL = `COALESCE((SELECT array_agg(ma.attachment_id::text ORDER BY ma.position)
FROM message_attachments ma WHERE ma.message_id = m.id), '{}')`
func (s *Store) PurgeIdempotency(ctx context.Context, ttl time.Duration) (int64, error) {
tag, err := s.pool.Exec(ctx,
`DELETE FROM message_idempotency WHERE created_at < now() - make_interval(secs => $1)`, ttl.Seconds())
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) {
var (
rows pgx.Rows
err error
)
if cur == nil {
rows, err = s.pool.Query(ctx,
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
FROM messages m
WHERE m.chat_id = $1
ORDER BY m.created_at, m.id LIMIT $2`, chatID, limit+1)
} else {
rows, err = s.pool.Query(ctx,
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
FROM messages m
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3::uuid)
ORDER BY m.created_at, m.id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1)
}
if err != nil {
return nil, false, err
}
defer rows.Close()
out := make([]*model.Message, 0, limit+1)
for rows.Next() {
m := &model.Message{}
var created time.Time
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil {
return nil, false, err
}
m.CreatedAt = utc(created)
out = append(out, m)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
more := len(out) > limit
if more {
out = out[:limit]
}
return out, more, nil
}
+365
View File
@@ -0,0 +1,365 @@
package worker
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"math/rand/v2"
"os"
"strings"
"sync"
"time"
"github.com/google/uuid"
amqp "github.com/rabbitmq/amqp091-go"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/imaging"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/store"
)
type Config struct {
BrokerURL string
Concurrency int
MaxAttempts int
JobTimeout time.Duration
Lease time.Duration
BackoffBase time.Duration
BackoffMax time.Duration
StagingRetain time.Duration
PendingGrace time.Duration
FaultInjection bool
Limits imaging.Limits
SweepEvery time.Duration
}
type Worker struct {
cfg Config
st *store.Store
s3 *objstore.Store
log *slog.Logger
owner string
inFly sync.WaitGroup
}
func New(cfg Config, st *store.Store, s3 *objstore.Store, log *slog.Logger) *Worker {
host, _ := os.Hostname()
return &Worker{cfg: cfg, st: st, s3: s3, log: log, owner: host + "/" + uuid.NewString()[:8]}
}
func (w *Worker) Run(ctx context.Context) error {
go w.sweeper(ctx)
for ctx.Err() == nil {
if err := w.consume(ctx); err != nil && ctx.Err() == nil {
w.log.Warn("consumer stopped, reconnecting", "error", err.Error())
select {
case <-ctx.Done():
case <-time.After(2 * time.Second):
}
}
}
w.inFly.Wait()
return nil
}
func (w *Worker) consume(ctx context.Context) error {
conn, err := broker.Dial(w.cfg.BrokerURL)
if err != nil {
return err
}
defer conn.Close()
ch, err := conn.Channel()
if err != nil {
return err
}
if err := broker.Declare(ch); err != nil {
return err
}
if err := ch.Qos(w.cfg.Concurrency, 0, false); err != nil {
return err
}
deliveries, err := ch.ConsumeWithContext(ctx, broker.Queue, w.owner, false, false, false, false, nil)
if err != nil {
return err
}
w.log.Info("worker consuming", "queue", broker.Queue, "concurrency", w.cfg.Concurrency, "owner", w.owner)
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
var wg sync.WaitGroup
for i := 0; i < w.cfg.Concurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for d := range deliveries {
w.inFly.Add(1)
w.handle(ctx, d)
w.inFly.Done()
}
}()
}
select {
case <-ctx.Done():
_ = ch.Cancel(w.owner, false)
wg.Wait()
return nil
case e := <-closed:
wg.Wait()
if e == nil {
return errors.New("broker connection closed")
}
return e
}
}
func (w *Worker) fault(job *store.JobWork, point string) bool {
return w.cfg.FaultInjection && strings.HasPrefix(job.Filename, "fault-"+point)
}
func (w *Worker) handle(ctx context.Context, d amqp.Delivery) {
var msg model.JobMessage
if err := json.Unmarshal(d.Body, &msg); err != nil || msg.SchemaVersion != 1 || msg.Type != model.JobType ||
uuid.Validate(msg.JobID) != nil || uuid.Validate(msg.AttachmentID) != nil || msg.Attempt < 1 || msg.PipelineVersion < 1 {
w.log.Error("malformed job message -> DLQ", "message_id", d.MessageId)
_ = d.Nack(false, false)
return
}
log := w.log.With("job_id", msg.JobID, "attachment_id", msg.AttachmentID, "attempt", msg.Attempt, "redelivered", d.Redelivered)
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
job, res, err := w.st.ClaimJob(cctx, msg.JobID, msg.Attempt, w.owner, w.cfg.Lease, d.Redelivered)
cancel()
if err != nil {
log.Warn("claim failed, requeue", "error", err.Error())
time.Sleep(2 * time.Second)
_ = d.Nack(false, true)
return
}
if res != store.Claimed {
log.Info("job delivery is a no-op", "result", string(res))
_ = d.Ack(false)
return
}
if job.AttachmentID != msg.AttachmentID {
log.Error("message attachment_id does not match job")
_ = d.Ack(false)
return
}
if w.fault(job, "hang") && job.Attempt == 1 {
log.Warn("FAULT: hang without heartbeat", "for", (3 * w.cfg.Lease).String())
time.Sleep(3 * w.cfg.Lease)
}
start := time.Now()
variants, perr := w.process(ctx, job, d.Redelivered, log)
if perr == nil {
cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ok, err := w.st.CompleteJob(cctx, job, variants)
cancel()
if err != nil {
log.Warn("result commit failed, requeue", "error", err.Error())
time.Sleep(2 * time.Second)
_ = d.Nack(false, true)
return
}
if !ok {
log.Info("stale attempt finished after takeover; result already owned by newer attempt")
} else {
log.Info("job ready", "ms", time.Since(start).Milliseconds())
w.dropStaging(job, log)
}
if w.fault(job, "worker-before-ack") && !d.Redelivered {
log.Warn("FAULT: exit after COMMIT before ack")
os.Exit(42)
}
_ = d.Ack(false)
return
}
code, permanent := classify(perr)
backoff := w.backoff(job.Attempt)
cctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
status, err := w.st.FailOrRetry(cctx, job.JobID, job.Attempt, code, safeDetail(perr), permanent, backoff, w.owner)
cancel()
if err != nil {
log.Warn("failure bookkeeping failed, requeue", "error", err.Error())
time.Sleep(2 * time.Second)
_ = d.Nack(false, true)
return
}
log.Warn("job attempt failed", "error_code", code, "permanent", permanent, "next", status, "backoff_ms", backoff.Milliseconds())
_ = d.Ack(false)
}
func (w *Worker) process(ctx context.Context, job *store.JobWork, redelivered bool, log *slog.Logger) ([]model.Variant, error) {
pctx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
defer cancel()
hbDone := make(chan struct{})
defer close(hbDone)
go func() {
t := time.NewTicker(w.cfg.Lease / 3)
defer t.Stop()
for {
select {
case <-hbDone:
return
case <-pctx.Done():
return
case <-t.C:
c, cc := context.WithTimeout(context.Background(), 3*time.Second)
_, _ = w.st.ExtendLease(c, job.JobID, job.Attempt, w.owner, w.cfg.Lease)
cc()
}
}
}()
if w.fault(job, "flaky") && job.Attempt <= w.cfg.MaxAttempts {
return nil, transientErr("injected_transient", "flaky fault until replay")
}
if w.fault(job, "transient") && job.Attempt < 3 {
return nil, transientErr("injected_transient", "first two attempts fail")
}
data, err := w.readOriginal(pctx, job)
if err != nil {
return nil, err
}
sum := sha256.Sum256(data)
if hex.EncodeToString(sum[:]) != job.SHA256 || int64(len(data)) != job.SizeBytes {
return nil, &imaging.PermanentError{Code: "processing_failed", Detail: "staged original checksum mismatch"}
}
outs, err := imaging.Process(pctx, data, job.ContentType, w.cfg.Limits)
if err != nil {
return nil, err
}
if err := w.s3.Put(pctx, job.ObjectKey, data, job.ContentType); err != nil {
return nil, transientErr("storage_unavailable", err.Error())
}
variants := make([]model.Variant, 0, len(outs))
for _, o := range outs {
key := fmt.Sprintf("variants/%s/%s/v%d/%s.%s", job.ChatID, job.AttachmentID, job.PipelineVersion, o.Name, o.Ext)
if err := w.s3.Put(pctx, key, o.Data, o.ContentType); err != nil {
return nil, transientErr("storage_unavailable", err.Error())
}
vs := sha256.Sum256(o.Data)
variants = append(variants, model.Variant{Name: o.Name, ContentType: o.ContentType, SizeBytes: int64(len(o.Data)),
Width: o.Width, Height: o.Height, SHA256: hex.EncodeToString(vs[:]), ObjectKey: key})
}
if w.fault(job, "worker-after-effect") && !redelivered {
log.Warn("FAULT: exit after S3 writes before COMMIT")
os.Exit(42)
}
return variants, nil
}
func (w *Worker) readOriginal(ctx context.Context, job *store.JobWork) ([]byte, error) {
if job.StagingKey != "" {
data, err := w.s3.ReadAll(ctx, job.StagingKey, w.cfg.Limits.MaxBytes)
if err == nil {
return data, nil
}
if !objstore.IsNotFound(err) {
return nil, transientErr("storage_unavailable", err.Error())
}
}
data, err := w.s3.ReadAll(ctx, job.ObjectKey, w.cfg.Limits.MaxBytes)
if objstore.IsNotFound(err) {
return nil, &imaging.PermanentError{Code: "processing_failed", Detail: "original is missing"}
}
if err != nil {
return nil, transientErr("storage_unavailable", err.Error())
}
return data, nil
}
func (w *Worker) dropStaging(job *store.JobWork, log *slog.Logger) {
if job.StagingKey == "" {
return
}
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := w.s3.Remove(c, job.StagingKey); err != nil {
log.Warn("staging cleanup deferred to sweeper", "error", err.Error())
return
}
_ = w.st.ClearStaging(c, job.AttachmentID)
}
func (w *Worker) backoff(attempt int) time.Duration {
d := w.cfg.BackoffBase << min(attempt-1, 20)
if d > w.cfg.BackoffMax || d <= 0 {
d = w.cfg.BackoffMax
}
return d/2 + time.Duration(rand.Int64N(int64(d/2)+1))
}
type transient struct{ code, detail string }
func (e *transient) Error() string { return e.code + ": " + e.detail }
func transientErr(code, detail string) error { return &transient{code, detail} }
func classify(err error) (string, bool) {
var p *imaging.PermanentError
var t *transient
switch {
case errors.As(err, &p):
return p.Code, true
case errors.As(err, &t):
return t.code, false
case errors.Is(err, context.DeadlineExceeded):
return "processing_timeout", false
}
return "processing_failed", false
}
func safeDetail(err error) string {
s := err.Error()
if i := strings.Index(s, "http"); i >= 0 {
s = s[:i] + "[redacted]"
}
if len(s) > 300 {
s = s[:300]
}
return s
}
func (w *Worker) sweeper(ctx context.Context) {
t := time.NewTicker(w.cfg.SweepEvery)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
}
c, cancel := context.WithTimeout(ctx, 30*time.Second)
if n, err := w.st.RecoverExpiredLeases(c, 100); err == nil && n > 0 {
w.log.Warn("expired leases recovered", "count", n)
}
refs, err := w.st.StaleStaging(c, w.cfg.PendingGrace, w.cfg.StagingRetain, 100)
if err == nil {
for _, r := range refs {
if err := w.s3.Remove(c, r.StagingKey); err != nil && !objstore.IsNotFound(err) {
continue
}
if err := w.st.ForgetStaging(c, r); err == nil {
w.log.Info("staging cleaned", "attachment_id", r.AttachmentID, "status", r.Status)
}
}
}
if n, err := w.st.PurgeOutbox(c, 24*time.Hour); err == nil && n > 0 {
w.log.Info("published outbox rows purged", "count", n)
}
cancel()
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

+34
View File
@@ -0,0 +1,34 @@
"""Synthetic PNG builders for negative/limit tests (stdlib only, nothing stored in Git)."""
import os
import struct
import zlib
def chunk(kind, data):
body = kind + data
return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff)
def png(width, height, raw=None, level=6, color_type=2):
"""RGB (color_type=2) PNG; raw is the filtered scanline stream (default: random noise)."""
if raw is None:
row = 1 + 3 * width
raw = b''.join(b'\x00' + os.urandom(row - 1) for _ in range(height))
ihdr = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) +
chunk(b'IDAT', zlib.compress(raw, level)) + chunk(b'IEND', b''))
def near_limit_png(limit=10 * 1024 * 1024):
"""Valid noise PNG a little below the byte limit (stored deflate blocks)."""
width = 1000
height = (limit - 64 * 1024) // (1 + 3 * width)
data = png(width, height, level=0)
assert len(data) <= limit, len(data)
return data
def bomb_png(width=20000, height=20000):
"""Header declares 400M pixels, the compressed payload is tiny."""
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) +
chunk(b'IDAT', zlib.compress(b'\x00' * 4096, 9)) + chunk(b'IEND', b''))